Tempo: Search for Traces by querying Loki directly from Tempo (#33308)

* Loki query from Tempo UI - add query type selector to tempo - introduce linkedDatasource concept that runs queries on behalf of another datasource - Tempo uses Loki's query field and Loki's derived fields to find a trace matcher - Tempo uses the trace-to-logs mechanism to determine which dataource is linked Loki data loads successfully via tempo Extracted result transformers Skip null values Show trace on list id click Query type selector Use linked field trace regexp * Review feedback
2025-01-16 03:32:37 -06:00 · 2021-05-10 17:12:19 +02:00 · 2021-05-10 17:12:19 +02:00 · 59c754823f
commit 59c754823f
parent da13f88862
5 changed files with 374 additions and 97 deletions
--- a/public/app/plugins/datasource/loki/components/LokiQueryField.tsx
+++ b/public/app/plugins/datasource/loki/components/LokiQueryField.tsx
@ -92,7 +92,7 @@ export class LokiQueryField extends React.PureComponent<LokiQueryFieldProps, Lok
    ];
  }

-  async componentDidUpdate() {
+  async componentDidMount() {
    await this.props.datasource.languageProvider.start();
    this.setState({ labelsLoaded: true });
  }
--- a/public/app/plugins/datasource/tempo/QueryField.tsx
+++ b/public/app/plugins/datasource/tempo/QueryField.tsx
@ -1,35 +1,124 @@
-import { ExploreQueryFieldProps } from '@grafana/data';
+import { DataQuery, DataSourceApi, ExploreQueryFieldProps } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
-import { LegacyForms } from '@grafana/ui';
+import { getDataSourceSrv } from '@grafana/runtime';
+import { InlineField, InlineFieldRow, InlineLabel, LegacyForms, RadioButtonGroup } from '@grafana/ui';
+import { TraceToLogsOptions } from 'app/core/components/TraceToLogsSettings';
 import React from 'react';
-import { TempoDatasource, TempoQuery } from './datasource';
+import { LokiQueryField } from '../loki/components/LokiQueryField';
+import { TempoDatasource, TempoQuery, TempoQueryType } from './datasource';

 type Props = ExploreQueryFieldProps<TempoDatasource, TempoQuery>;
-export class TempoQueryField extends React.PureComponent<Props> {
-  render() {
+const DEFAULT_QUERY_TYPE: TempoQueryType = 'traceId';
+interface State {
+  linkedDatasource?: DataSourceApi;
+}
+export class TempoQueryField extends React.PureComponent<Props, State> {
+  state = {
+    linkedDatasource: undefined,
+  };
+  linkedQuery: DataQuery;
+  constructor(props: Props) {
+    super(props);
+    this.linkedQuery = { refId: 'linked' };
+  }
+
+  async componentDidMount() {
+    const { datasource } = this.props;
+    // Find query field from linked datasource
+    const tracesToLogsOptions: TraceToLogsOptions = datasource.tracesToLogs || {};
+    const linkedDatasourceUid = tracesToLogsOptions.datasourceUid;
+    if (linkedDatasourceUid) {
+      const dsSrv = getDataSourceSrv();
+      const linkedDatasource = await dsSrv.get(linkedDatasourceUid);
+      this.setState({
+        linkedDatasource,
+      });
+    }
+  }
+
+  onChangeLinkedQuery = (value: DataQuery) => {
    const { query, onChange } = this.props;
+    this.linkedQuery = value;
+    onChange({
+      ...query,
+      linkedQuery: this.linkedQuery,
+    });
+  };
+
+  onRunLinkedQuery = () => {
+    this.props.onRunQuery();
+  };
+
+  render() {
+    const { query, onChange, range } = this.props;
+    const { linkedDatasource } = this.state;
+
+    const absoluteTimeRange = { from: range!.from!.valueOf(), to: range!.to!.valueOf() }; // Range here is never optional

    return (
-      <LegacyForms.FormField
-        label="Trace ID"
-        labelWidth={4}
-        inputEl={
-          <div className="slate-query-field__wrapper">
-            <div className="slate-query-field" aria-label={selectors.components.QueryField.container}>
-              <input
-                style={{ width: '100%' }}
-                value={query.query || ''}
-                onChange={(e) =>
-                  onChange({
-                    ...query,
-                    query: e.currentTarget.value,
-                  })
-                }
-              />
-            </div>
-          </div>
-        }
-      />
+      <>
+        <InlineFieldRow>
+          <InlineField label="Query type">
+            <RadioButtonGroup<TempoQueryType>
+              options={[
+                { value: 'search', label: 'Search' },
+                { value: 'traceId', label: 'TraceID' },
+              ]}
+              value={query.queryType || DEFAULT_QUERY_TYPE}
+              onChange={(v) =>
+                onChange({
+                  ...query,
+                  queryType: v,
+                })
+              }
+              size="md"
+            />
+          </InlineField>
+        </InlineFieldRow>
+        {query.queryType === 'search' && linkedDatasource && (
+          <>
+            <InlineLabel>
+              Tempo uses {((linkedDatasource as unknown) as DataSourceApi).name} to find traces.
+            </InlineLabel>
+
+            <LokiQueryField
+              datasource={linkedDatasource!}
+              onChange={this.onChangeLinkedQuery}
+              onRunQuery={this.onRunLinkedQuery}
+              query={this.linkedQuery as any}
+              history={[]}
+              absoluteRange={absoluteTimeRange}
+            />
+          </>
+        )}
+        {query.queryType === 'search' && !linkedDatasource && (
+          <div className="text-warning">Please set up a Traces-to-logs datasource in the datasource settings.</div>
+        )}
+        {query.queryType !== 'search' && (
+          <LegacyForms.FormField
+            label="Trace ID"
+            labelWidth={4}
+            inputEl={
+              <div className="slate-query-field__wrapper">
+                <div className="slate-query-field" aria-label={selectors.components.QueryField.container}>
+                  <input
+                    style={{ width: '100%' }}
+                    value={query.query || ''}
+                    onChange={(e) =>
+                      onChange({
+                        ...query,
+                        query: e.currentTarget.value,
+                        queryType: 'traceId',
+                        linkedQuery: undefined,
+                      })
+                    }
+                  />
+                </div>
+              </div>
+            }
+          />
+        )}
+      </>
    );
  }
 }
--- a/public/app/plugins/datasource/tempo/datasource.ts
+++ b/public/app/plugins/datasource/tempo/datasource.ts
@ -1,51 +1,93 @@
 import {
-  ArrayVector,
-  DataFrame,
  DataQuery,
  DataQueryRequest,
  DataQueryResponse,
+  DataSourceApi,
  DataSourceInstanceSettings,
-  Field,
-  FieldType,
-  MutableDataFrame,
 } from '@grafana/data';
 import { DataSourceWithBackend } from '@grafana/runtime';
-import { Observable } from 'rxjs';
+import { TraceToLogsData, TraceToLogsOptions } from 'app/core/components/TraceToLogsSettings';
+import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
+import { merge, Observable, throwError } from 'rxjs';
 import { map } from 'rxjs/operators';
-import { createGraphFrames } from './graphTransform';
+import { LokiOptions } from '../loki/types';
+import { transformTrace, transformTraceList } from './resultTransformer';
+
+export type TempoQueryType = 'search' | 'traceId';

 export type TempoQuery = {
  query: string;
+  // Query to find list of traces, e.g., via Loki
+  linkedQuery?: DataQuery;
+  queryType: TempoQueryType;
 } & DataQuery;

-export class TempoDatasource extends DataSourceWithBackend<TempoQuery> {
-  constructor(instanceSettings: DataSourceInstanceSettings) {
+export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TraceToLogsData> {
+  tracesToLogs: TraceToLogsOptions;
+  linkedDatasource: DataSourceApi;
+  constructor(instanceSettings: DataSourceInstanceSettings<TraceToLogsData>) {
    super(instanceSettings);
+    this.tracesToLogs = instanceSettings.jsonData.tracesToLogs || {};
+    if (this.tracesToLogs.datasourceUid) {
+      this.linkDatasource();
+    }
+  }
+
+  async linkDatasource() {
+    const dsSrv = getDatasourceSrv();
+    this.linkedDatasource = await dsSrv.get(this.tracesToLogs.datasourceUid);
  }

  query(options: DataQueryRequest<TempoQuery>): Observable<DataQueryResponse> {
-    return super.query(options).pipe(
-      map((response) => {
-        if (response.error) {
-          return response;
-        }
-        // We need to parse some of the fields which contain stringified json.
-        // Seems like we can't just map the values as the frame we got from backend has some default processing
-        // and will stringify the json back when we try to set it. So we create a new field and swap it instead.
-        const frame: DataFrame = response.data[0];
-
-        if (!frame) {
-          return emptyDataQueryResponse;
-        }
-
-        parseJsonFields(frame);
-
-        return {
-          ...response,
-          data: [...response.data, ...createGraphFrames(frame)],
-        };
-      })
+    const subQueries: Array<Observable<DataQueryResponse>> = [];
+    const filteredTargets = options.targets.filter((target) => !target.hide);
+    const searchTargets = filteredTargets.filter((target) => target.queryType === 'search');
+    const traceTargets = filteredTargets.filter(
+      (target) => target.queryType === 'traceId' || target.queryType === undefined
    );
+
+    // Run search queries on linked datasource
+    if (this.linkedDatasource && searchTargets.length > 0) {
+      // Wrap linked query into a data request based on original request
+      const linkedRequest: DataQueryRequest = { ...options, targets: searchTargets.map((t) => t.linkedQuery!) };
+      // Find trace matchers in derived fields of the linked datasource that's identical to this datasource
+      const settings: DataSourceInstanceSettings<LokiOptions> = (this.linkedDatasource as any).instanceSettings;
+      const traceLinkMatcher: string[] =
+        settings.jsonData.derivedFields
+          ?.filter((field) => field.datasourceUid === this.uid && field.matcherRegex)
+          .map((field) => field.matcherRegex) || [];
+      if (!traceLinkMatcher || traceLinkMatcher.length === 0) {
+        subQueries.push(
+          throwError(
+            'No Loki datasource configured for search. Set up Derived Fields for traces in a Loki datasource settings and link it to this Tempo datasource.'
+          )
+        );
+      } else {
+        subQueries.push(
+          (this.linkedDatasource.query(linkedRequest) as Observable<DataQueryResponse>).pipe(
+            map((response) =>
+              response.error ? response : transformTraceList(response, this.uid, this.name, traceLinkMatcher)
+            )
+          )
+        );
+      }
+    }
+
+    if (traceTargets.length > 0) {
+      const traceRequest: DataQueryRequest<TempoQuery> = { ...options, targets: traceTargets };
+      subQueries.push(
+        super.query(traceRequest).pipe(
+          map((response) => {
+            if (response.error) {
+              return response;
+            }
+            return transformTrace(response);
+          })
+        )
+      );
+    }
+
+    return merge(...subQueries);
  }

  async testDatasource(): Promise<any> {
@ -62,44 +104,3 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery> {
    return query.query;
  }
 }
-
-/**
- * Change fields which are json string into JS objects. Modifies the frame in place.
- */
-function parseJsonFields(frame: DataFrame) {
-  for (const fieldName of ['serviceTags', 'logs', 'tags']) {
-    const field = frame.fields.find((f) => f.name === fieldName);
-    if (field) {
-      const fieldIndex = frame.fields.indexOf(field);
-      const values = new ArrayVector();
-      const newField: Field = {
-        ...field,
-        values,
-        type: FieldType.other,
-      };
-
-      for (let i = 0; i < field.values.length; i++) {
-        const value = field.values.get(i);
-        values.set(i, value === '' ? undefined : JSON.parse(value));
-      }
-      frame.fields[fieldIndex] = newField;
-    }
-  }
-}
-
-const emptyDataQueryResponse = {
-  data: [
-    new MutableDataFrame({
-      fields: [
-        {
-          name: 'trace',
-          type: FieldType.trace,
-          values: [],
-        },
-      ],
-      meta: {
-        preferredVisualisationType: 'trace',
-      },
-    }),
-  ],
-};
--- a/public/app/plugins/datasource/tempo/resultTransformer.test.ts
+++ b/public/app/plugins/datasource/tempo/resultTransformer.test.ts
@ -0,0 +1,37 @@
+import { FieldType, MutableDataFrame } from '@grafana/data';
+import { createTableFrame } from './resultTransformer';
+
+describe('transformTraceList()', () => {
+  const lokiDataFrame = new MutableDataFrame({
+    fields: [
+      {
+        name: 'ts',
+        type: FieldType.time,
+        values: ['2020-02-12T15:05:14.265Z', '2020-02-12T15:05:15.265Z', '2020-02-12T15:05:16.265Z'],
+      },
+      {
+        name: 'line',
+        type: FieldType.string,
+        values: [
+          't=2020-02-12T15:04:51+0000 lvl=info msg="Starting Grafana" logger=server',
+          't=2020-02-12T15:04:52+0000 lvl=info msg="Starting Grafana" logger=server traceID=asdfa1234',
+          't=2020-02-12T15:04:53+0000 lvl=info msg="Starting Grafana" logger=server traceID=asdf88',
+        ],
+      },
+    ],
+    meta: {
+      preferredVisualisationType: 'table',
+    },
+  });
+
+  test('extracts traceIDs from log lines', () => {
+    const frame = createTableFrame(lokiDataFrame, 't1', 'tempo', ['traceID=(\\w+)', 'traceID=(\\w\\w)']);
+    expect(frame.fields[0].name).toBe('Time');
+    expect(frame.fields[0].values.get(0)).toBe('2020-02-12T15:05:15.265Z');
+    expect(frame.fields[1].name).toBe('traceID');
+    expect(frame.fields[1].values.get(0)).toBe('asdfa1234');
+    // Second match in new line
+    expect(frame.fields[0].values.get(1)).toBe('2020-02-12T15:05:15.265Z');
+    expect(frame.fields[1].values.get(1)).toBe('as');
+  });
+});
--- a/public/app/plugins/datasource/tempo/resultTransformer.ts
+++ b/public/app/plugins/datasource/tempo/resultTransformer.ts
@ -0,0 +1,150 @@
+import { DataQueryResponse, ArrayVector, DataFrame, Field, FieldType, MutableDataFrame } from '@grafana/data';
+import { createGraphFrames } from './graphTransform';
+
+export function createTableFrame(
+  logsFrame: DataFrame,
+  datasourceUid: string,
+  datasourceName: string,
+  traceRegexs: string[]
+): DataFrame {
+  const tableFrame = new MutableDataFrame({
+    fields: [
+      {
+        name: 'Time',
+        type: FieldType.time,
+      },
+      {
+        name: 'traceID',
+        type: FieldType.string,
+        config: {
+          displayNameFromDS: 'Trace ID',
+          links: [
+            {
+              title: 'Click to open trace ${__value.raw}',
+              url: '',
+              internal: {
+                datasourceUid,
+                datasourceName,
+                query: {
+                  query: '${__value.raw}',
+                },
+              },
+            },
+          ],
+        },
+      },
+      {
+        name: 'Message',
+        type: FieldType.string,
+      },
+    ],
+    meta: {
+      preferredVisualisationType: 'table',
+    },
+  });
+
+  if (!logsFrame || traceRegexs.length === 0) {
+    return tableFrame;
+  }
+
+  const timeField = logsFrame.fields.find((f) => f.type === FieldType.time);
+
+  // Going through all string fields to look for trace IDs
+  for (let field of logsFrame.fields) {
+    let hasMatch = false;
+    if (field.type === FieldType.string) {
+      const values = field.values.toArray();
+      for (let i = 0; i < values.length; i++) {
+        const line = values[i];
+        if (line) {
+          for (let traceRegex of traceRegexs) {
+            const match = (line as string).match(traceRegex);
+            if (match) {
+              const traceId = match[1];
+              const time = timeField ? timeField.values.get(i) : null;
+              tableFrame.fields[0].values.add(time);
+              tableFrame.fields[1].values.add(traceId);
+              tableFrame.fields[2].values.add(line);
+              hasMatch = true;
+            }
+          }
+        }
+      }
+    }
+    if (hasMatch) {
+      break;
+    }
+  }
+
+  return tableFrame;
+}
+
+export function transformTraceList(
+  response: DataQueryResponse,
+  datasourceId: string,
+  datasourceName: string,
+  traceRegexs: string[]
+): DataQueryResponse {
+  const frame = createTableFrame(response.data[0], datasourceId, datasourceName, traceRegexs);
+  response.data[0] = frame;
+  return response;
+}
+
+export function transformTrace(response: DataQueryResponse): DataQueryResponse {
+  // We need to parse some of the fields which contain stringified json.
+  // Seems like we can't just map the values as the frame we got from backend has some default processing
+  // and will stringify the json back when we try to set it. So we create a new field and swap it instead.
+  const frame: DataFrame = response.data[0];
+
+  if (!frame) {
+    return emptyDataQueryResponse;
+  }
+
+  parseJsonFields(frame);
+
+  return {
+    ...response,
+    data: [...response.data, ...createGraphFrames(frame)],
+  };
+}
+
+/**
+ * Change fields which are json string into JS objects. Modifies the frame in place.
+ */
+function parseJsonFields(frame: DataFrame) {
+  for (const fieldName of ['serviceTags', 'logs', 'tags']) {
+    const field = frame.fields.find((f) => f.name === fieldName);
+    if (field) {
+      const fieldIndex = frame.fields.indexOf(field);
+      const values = new ArrayVector();
+      const newField: Field = {
+        ...field,
+        values,
+        type: FieldType.other,
+      };
+
+      for (let i = 0; i < field.values.length; i++) {
+        const value = field.values.get(i);
+        values.set(i, value === '' ? undefined : JSON.parse(value));
+      }
+      frame.fields[fieldIndex] = newField;
+    }
+  }
+}
+
+const emptyDataQueryResponse = {
+  data: [
+    new MutableDataFrame({
+      fields: [
+        {
+          name: 'trace',
+          type: FieldType.trace,
+          values: [],
+        },
+      ],
+      meta: {
+        preferredVisualisationType: 'trace',
+      },
+    }),
+  ],
+};