Tempo: Integrate TraceQL API (#56867)

2025-02-16 18:34:52 -06:00 · 2022-10-20 15:44:59 +01:00 · 2022-10-20 15:44:59 +01:00 · 84a5ced72a
commit 84a5ced72a
parent 0232927591
9 changed files with 418 additions and 119 deletions
--- a/devenv/docker/blocks/tempo/docker-compose.yaml
+++ b/devenv/docker/blocks/tempo/docker-compose.yaml
@ -68,7 +68,7 @@ services:

  tempo:
    image: grafana/tempo:latest
-    command: 
+    command:
      - --config.file=/etc/tempo.yaml
      - --search.enabled=true
    volumes:
--- a/devenv/docker/blocks/tempo/tempo.yaml
+++ b/devenv/docker/blocks/tempo/tempo.yaml
@ -41,6 +41,10 @@ metrics_generator:
      - url: http://prometheus:9090/api/v1/write
        send_exemplars: true

+query_frontend:
+  search:
+    max_duration: 0 # allow searches >1h
+
 storage:
  trace:
    backend: local                     # backend configuration to use
@ -48,6 +52,7 @@ storage:
      bloom_filter_false_positive: .05 # bloom filter false positive rate.  lower values create larger filters but fewer false positives
      index_downsample_bytes: 1000     # number of bytes per index record
      encoding: zstd                   # block encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2
+      version: vParquet
    wal:
      path: /tmp/tempo/wal             # where to store the the wal locally
      encoding: snappy                 # wal encoding/compression.  options: none, gzip, lz4-64k, lz4-256k, lz4-1M, lz4, snappy, zstd, s2
--- a/public/app/plugins/datasource/tempo/datasource.ts
+++ b/public/app/plugins/datasource/tempo/datasource.ts
@ -51,7 +51,6 @@ import {
  createTableFrameFromSearch,
  createTableFrameFromTraceQlQuery,
 } from './resultTransformer';
-import { mockedSearchResponse } from './traceql/mockedSearchResponse';
 import { SearchQueryParams, TempoQuery, TempoJsonData } from './types';

 export const DEFAULT_LIMIT = 20;
@ -174,9 +173,21 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
        });

        subQueries.push(
-          of({
-            data: [createTableFrameFromTraceQlQuery(mockedSearchResponse().traces, this.instanceSettings)],
-          })
+          this._request('/api/search', {
+            q: targets.traceql[0].query,
+            limit: options.targets[0].limit,
+            start: 0, // Currently the API doesn't return traces when using the 'From' time selected in Explore
+            end: options.range.to.unix(),
+          }).pipe(
+            map((response) => {
+              return {
+                data: [createTableFrameFromTraceQlQuery(response.data.traces, this.instanceSettings)],
+              };
+            }),
+            catchError((error) => {
+              return of({ error: { message: error.data.message }, data: [] });
+            })
+          )
        );
      } catch (error) {
        return of({ error: { message: error instanceof Error ? error.message : 'Unknown error occurred' }, data: [] });
--- a/public/app/plugins/datasource/tempo/resultTransformer.test.ts
+++ b/public/app/plugins/datasource/tempo/resultTransformer.test.ts
@ -1,14 +1,28 @@
 import { collectorTypes } from '@opentelemetry/exporter-collector';

-import { FieldType, MutableDataFrame, PluginType, DataSourceInstanceSettings, dateTime } from '@grafana/data';
+import {
+  FieldType,
+  MutableDataFrame,
+  PluginType,
+  DataSourceInstanceSettings,
+  dateTime,
+  ArrayVector,
+} from '@grafana/data';

-import { createTableFrame, transformToOTLP, transformFromOTLP, createTableFrameFromSearch } from './resultTransformer';
+import {
+  createTableFrame,
+  transformToOTLP,
+  transformFromOTLP,
+  createTableFrameFromSearch,
+  createTableFrameFromTraceQlQuery,
+} from './resultTransformer';
 import {
  badOTLPResponse,
  otlpDataFrameToResponse,
  otlpDataFrameFromResponse,
  otlpResponse,
  tempoSearchResponse,
+  traceQlResponse,
 } from './testResponse';
 import { TraceSearchMetadata } from './types';

@ -111,6 +125,45 @@ describe('createTableFrameFromSearch()', () => {
  });
 });

+describe('createTableFrameFromTraceQlQuery()', () => {
+  test('transforms TraceQL response to DataFrame', () => {
+    const frame = createTableFrameFromTraceQlQuery(traceQlResponse.traces, defaultSettings);
+    // Trace ID field
+    expect(frame.fields[0].name).toBe('traceID');
+    expect(frame.fields[0].values.get(0)).toBe('b1586c3c8c34d');
+    expect(frame.fields[0].config.unit).toBe('string');
+    expect(frame.fields[0].values).toBeInstanceOf(ArrayVector);
+    // There should be a traceIdHidden field which is hidden
+    expect(frame.fields[1].name).toBe('traceIdHidden');
+    expect(frame.fields[1].config).toBeDefined();
+    expect(frame.fields[1].config.custom).toStrictEqual({ hidden: true });
+    expect(frame.fields[1].values).toBeInstanceOf(ArrayVector);
+    // Span ID field
+    expect(frame.fields[2].name).toBe('spanID');
+    expect(frame.fields[2].config.unit).toBe('string');
+    expect(frame.fields[2].values).toBeInstanceOf(ArrayVector);
+    // Trace name field
+    expect(frame.fields[3].name).toBe('traceName');
+    expect(frame.fields[3].type).toBe('string');
+    expect(frame.fields[3].values.get(0)).toBe('lb HTTP Client');
+    expect(frame.fields[3].values).toBeInstanceOf(ArrayVector);
+    // Start time field
+    expect(frame.fields[4].name).toBe('startTime');
+    expect(frame.fields[4].type).toBe('string');
+    expect(frame.fields[4].values.get(1)).toBe('2022-10-19 09:03:34');
+    expect(frame.fields[4].values).toBeInstanceOf(ArrayVector);
+    // Duration field
+    expect(frame.fields[5].name).toBe('duration');
+    expect(frame.fields[5].type).toBe('number');
+    expect(frame.fields[5].values.get(2)).toBe(6686000);
+    // There should be a field for each attribute
+    expect(frame.fields[6].name).toBe('http.method');
+    expect(frame.fields[6].values).toBeInstanceOf(ArrayVector);
+    expect(frame.fields[7].name).toBe('service.name');
+    expect(frame.fields[7].values).toBeInstanceOf(ArrayVector);
+  });
+});
+
 describe('transformFromOTLP()', () => {
  // Mock the console error so that running the test suite doesnt throw the error
  const origError = console.error;
--- a/public/app/plugins/datasource/tempo/resultTransformer.ts
+++ b/public/app/plugins/datasource/tempo/resultTransformer.ts
@ -577,7 +577,7 @@ export function createTableFrameFromSearch(data: TraceSearchMetadata[], instance
      },
      { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Trace name' } },
      { name: 'startTime', type: FieldType.string, config: { displayNameFromDS: 'Start time' } },
-      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ms' } },
+      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ns' } },
    ],
    meta: {
      preferredVisualisationType: 'table',
@ -609,7 +609,7 @@ function transformToTraceData(data: TraceSearchMetadata) {

  const traceStartTime = parseInt(data.startTimeUnixNano!, 10) / 1000000;

-  let startTime = dateTimeFormat(traceStartTime);
+  let startTime = !isNaN(traceStartTime) ? dateTimeFormat(traceStartTime) : '';

  if (Math.abs(differenceInHours(new Date(traceStartTime), Date.now())) <= 1) {
    startTime = formatDistance(new Date(traceStartTime), Date.now(), {
@ -621,7 +621,7 @@ function transformToTraceData(data: TraceSearchMetadata) {
  return {
    traceID: data.traceID,
    startTime: startTime,
-    duration: data.durationMs,
+    duration: data.durationMs?.toString(),
    traceName,
  };
 }
@ -654,12 +654,18 @@ export function createTableFrameFromTraceQlQuery(
          ],
        },
      },
+      {
+        name: 'traceIdHidden',
+        config: {
+          custom: { hidden: true },
+        },
+      },
      {
        name: 'spanID',
        type: FieldType.string,
        config: {
          unit: 'string',
-          displayNameFromDS: 'Span',
+          displayNameFromDS: 'Span ID',
          links: [
            {
              title: 'Span: ${__value.raw}',
@ -668,18 +674,18 @@ export function createTableFrameFromTraceQlQuery(
                datasourceUid: instanceSettings.uid,
                datasourceName: instanceSettings.name,
                query: {
-                  query: '${__value.raw}',
-                  queryType: 'spanId',
+                  query: '${__data.fields.traceIdHidden}',
+                  queryType: 'traceId',
                },
              },
            },
          ],
        },
      },
-      { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Name' } },
-      { name: 'attributes', type: FieldType.string, config: { displayNameFromDS: 'Attributes' } },
+      { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Trace name' } },
+      // { name: 'attributes', type: FieldType.string, config: { displayNameFromDS: 'Attributes' } },
      { name: 'startTime', type: FieldType.string, config: { displayNameFromDS: 'Start time' } },
-      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ms' } },
+      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ns' } },
    ],
    meta: {
      preferredVisualisationType: 'table',
@ -688,57 +694,71 @@ export function createTableFrameFromTraceQlQuery(
  if (!data?.length) {
    return frame;
  }
-  // Show the most recent traces
-  const traceData = data
+
+  const attributesAdded: string[] = [];
+
+  data.forEach((trace) => {
+    trace.spanSet?.spans.forEach((span) => {
+      span.attributes?.forEach((attr) => {
+        if (!attributesAdded.includes(attr.key)) {
+          frame.addField({ name: attr.key, type: FieldType.string, config: { displayNameFromDS: attr.key } });
+          attributesAdded.push(attr.key);
+        }
+      });
+    });
+  });
+
+  const tableRows = data
+    // Show the most recent traces
    .sort((a, b) => parseInt(b?.startTimeUnixNano!, 10) / 1000000 - parseInt(a?.startTimeUnixNano!, 10) / 1000000)
-    .reduce((list: TraceTableData[], t) => {
-      const firstSpanSet = t.spanSets?.[0];
-      const trace: TraceTableData = transformToTraceData(t);
-      trace.attributes = firstSpanSet?.attributes
-        .map((atr) => Object.keys(atr).map((key) => `${key} = ${atr[key]}`))
-        .join(', ');
-      list.push(trace);
-      firstSpanSet?.spans.forEach((span) => list.push(transformSpanToTraceData(span)));
-      return list;
+    .reduce((rows: TraceTableData[], trace) => {
+      const traceData: TraceTableData = transformToTraceData(trace);
+      rows.push(traceData);
+      trace.spanSet?.spans.forEach((span) => {
+        rows.push(transformSpanToTraceData(span, trace.traceID));
+      });
+      return rows;
    }, []);

-  for (const trace of traceData) {
-    frame.add(trace);
+  for (const row of tableRows) {
+    frame.add(row);
  }

  return frame;
 }

 interface TraceTableData {
+  [key: string]: string | number | undefined; // dynamic attribute name
  traceID?: string;
-  traceName: string;
  spanID?: string;
-  attributes?: string;
-  startTime: string;
-  duration: number;
+  //attributes?: string;
+  startTime?: string;
+  duration?: string;
 }

-function transformSpanToTraceData(data: Span): TraceTableData {
-  const traceStartTime = data.startTimeUnixNano / 1000000;
-  const traceEndTime = data.endTimeUnixNano / 1000000;
+function transformSpanToTraceData(span: Span, traceID: string): TraceTableData {
+  const spanStartTimeUnixMs = parseInt(span.startTimeUnixNano, 10) / 1000000;
+  let spanStartTime = dateTimeFormat(spanStartTimeUnixMs);

-  let startTime = dateTimeFormat(traceStartTime);
-
-  if (Math.abs(differenceInHours(new Date(traceStartTime), Date.now())) <= 1) {
-    startTime = formatDistance(new Date(traceStartTime), Date.now(), {
+  if (Math.abs(differenceInHours(new Date(spanStartTime), Date.now())) <= 1) {
+    spanStartTime = formatDistance(new Date(spanStartTime), Date.now(), {
      addSuffix: true,
      includeSeconds: true,
    });
  }

-  return {
-    traceID: undefined,
-    traceName: data.name,
-    spanID: data.spanId,
-    attributes: data.attributes?.map((atr) => Object.keys(atr).map((key) => `${key} = ${atr[key]}`)).join(', '),
-    startTime,
-    duration: traceEndTime - traceStartTime,
+  const data: TraceTableData = {
+    traceIdHidden: traceID,
+    spanID: span.spanID,
+    startTime: spanStartTime,
+    duration: span.durationNanos,
  };
+
+  span.attributes?.forEach((attr) => {
+    data[attr.key] = attr.value.stringValue;
+  });
+
+  return data;
 }

 const emptyDataQueryResponse = {
--- a/public/app/plugins/datasource/tempo/testResponse.ts
+++ b/public/app/plugins/datasource/tempo/testResponse.ts
@ -2204,6 +2204,273 @@ export const tempoSearchResponse = {
  },
 };

+export const traceQlResponse = {
+  traces: [
+    {
+      traceID: 'b1586c3c8c34d',
+      rootServiceName: 'lb',
+      rootTraceName: 'HTTP Client',
+      spanSet: {
+        spans: [
+          {
+            spanID: '162a4adae63b61f1',
+            startTimeUnixNano: '1666188214303201000',
+            durationNanos: '545000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'db',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '15991be3a92136e6',
+            startTimeUnixNano: '1666188214300239000',
+            durationNanos: '6686000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '5e91b69dc224c240',
+            startTimeUnixNano: '1666188214300647000',
+            durationNanos: '6043000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '29f218a50b00c306',
+            startTimeUnixNano: '1666188214297891000',
+            durationNanos: '8365000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'lb',
+                },
+              },
+            ],
+          },
+        ],
+        matched: 4,
+      },
+    },
+    {
+      traceID: '9161e77388f3e',
+      rootServiceName: 'lb',
+      rootTraceName: 'HTTP Client',
+      spanSet: {
+        spans: [
+          {
+            spanID: '3b9a5c222d3ddd8f',
+            startTimeUnixNano: '1666187875397721000',
+            durationNanos: '877000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'db',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '894d90db6b5807f',
+            startTimeUnixNano: '1666187875393293000',
+            durationNanos: '11073000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: 'd3284e9c5081aab',
+            startTimeUnixNano: '1666187875393897000',
+            durationNanos: '10133000',
+            attributes: [
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '454785498fc8b1aa',
+            startTimeUnixNano: '1666187875389957000',
+            durationNanos: '13953000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'lb',
+                },
+              },
+            ],
+          },
+        ],
+        matched: 4,
+      },
+    },
+    {
+      traceID: '480691f7c6f20',
+      rootServiceName: 'lb',
+      rootTraceName: 'HTTP Client',
+      spanSet: {
+        spans: [
+          {
+            spanID: '2ab970c9db57d100',
+            startTimeUnixNano: '1666186467658853000',
+            durationNanos: '436000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'db',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '3a4070e418857cbd',
+            startTimeUnixNano: '1666186467657066000',
+            durationNanos: '5503000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '7ddf87d7a3f864c8',
+            startTimeUnixNano: '1666186467657336000',
+            durationNanos: '5005000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'app',
+                },
+              },
+            ],
+          },
+          {
+            spanID: '241e9f31609056c5',
+            startTimeUnixNano: '1666186467655299000',
+            durationNanos: '6413000',
+            attributes: [
+              {
+                key: 'http.method',
+                value: {
+                  stringValue: 'GET',
+                },
+              },
+              {
+                key: 'service.name',
+                value: {
+                  stringValue: 'lb',
+                },
+              },
+            ],
+          },
+        ],
+        matched: 4,
+      },
+    },
+  ],
+  metrics: {
+    inspectedBlocks: 5,
+    totalBlockBytes: '9092129',
+  },
+};
+
 export const badOTLPResponse = {
  batches: [
    {
--- a/public/app/plugins/datasource/tempo/traceql/TempoQueryBuilderOptions.tsx
+++ b/public/app/plugins/datasource/tempo/traceql/TempoQueryBuilderOptions.tsx
@ -12,6 +12,10 @@ interface Props {
 }

 export const TempoQueryBuilderOptions = React.memo<Props>(({ onChange, query }) => {
+  if (!query.hasOwnProperty('limit')) {
+    query.limit = DEFAULT_LIMIT;
+  }
+
  const onLimitChange = (e: React.FormEvent<HTMLInputElement>) => {
    onChange({ ...query, limit: parseInt(e.currentTarget.value, 10) });
  };
--- a/public/app/plugins/datasource/tempo/traceql/mockedSearchResponse.ts
+++ b/public/app/plugins/datasource/tempo/traceql/mockedSearchResponse.ts
@ -1,62 +0,0 @@
-import { v4 as uuidv4 } from 'uuid';
-
-import { SearchResponse, Span, SpanKind, TraceSearchMetadata } from '../types';
-
-export const mockedSearchResponse = (): SearchResponse => {
-  const traces: TraceSearchMetadata[] = [];
-  const attributes = [
-    { 'http.status.code': '500' },
-    { 'http.status.code': '200' },
-    { 'http.status.code': '404' },
-    { job: '"test-job"' },
-    { job: '"main-job"' },
-    { job: '"long-job"' },
-    { error: '"lorem ipsum"' },
-    { error: '"something went wrong"' },
-  ];
-
-  const tracesCount = Math.random() * 20 + 20;
-  for (let i = 0; i < tracesCount; i++) {
-    const attr = Math.floor(Math.random() * attributes.length);
-    const startTime = (Date.now() - Math.random() * (i + 1) * 100000) * 1000000;
-    const t: TraceSearchMetadata = {
-      traceID: uuidv4().replace(/-/, '').substring(0, 16),
-      rootServiceName: 'service' + i,
-      rootTraceName: 'trace' + i,
-      startTimeUnixNano: startTime.toString(10),
-      durationMs: Math.random() * 1000,
-      spanSets: [],
-    };
-
-    const spanAttributes = [];
-    for (let k = 0; k < Math.random() * 2; k++) {
-      const newAttr = Math.floor(Math.random() * attributes.length);
-      if (newAttr !== attr) {
-        spanAttributes.push(attributes[newAttr]);
-      }
-    }
-
-    const spans: Span[] = [];
-    for (let j = 0; j < Math.random() * 3 + 1; j++) {
-      spans.push({
-        traceId: t.traceID,
-        spanId: uuidv4().replace(/-/, '').substring(0, 16),
-        name: uuidv4().replace(/-/, '').substring(0, 6),
-        startTimeUnixNano: startTime,
-        endTimeUnixNano: startTime + Math.random() * 10000000,
-        kind: SpanKind.INTERNAL,
-        attributes: spanAttributes,
-      });
-    }
-    t.spanSets!.push({ spans, attributes: [attributes[attr]] });
-    traces.push(t);
-  }
-
-  return {
-    traces,
-    metrics: {
-      inspectedTraces: tracesCount,
-      inspectedBytes: 83720,
-    },
-  };
-};
--- a/public/app/plugins/datasource/tempo/types.ts
+++ b/public/app/plugins/datasource/tempo/types.ts
@ -56,9 +56,9 @@ export type TraceSearchMetadata = {
  traceID: string;
  rootServiceName: string;
  rootTraceName: string;
-  startTimeUnixNano: string;
-  durationMs: number;
-  spanSets?: Spanset[];
+  startTimeUnixNano?: string;
+  durationMs?: number;
+  spanSet?: { spans: Span[] };
 };

 export type SearchMetrics = {
@ -81,15 +81,16 @@ export enum SpanKind {
 }

 export type Span = {
-  traceId: string;
-  spanId: string;
+  durationNanos: string;
+  traceId?: string;
+  spanID: string;
  traceState?: string;
  parentSpanId?: string;
-  name: string;
-  kind: SpanKind;
-  startTimeUnixNano: number;
-  endTimeUnixNano: number;
-  attributes?: KeyValue[];
+  name?: string;
+  kind?: SpanKind;
+  startTimeUnixNano: string;
+  endTimeUnixNano?: string;
+  attributes?: Array<{ key: string; value: { stringValue: string } }>;
  dropped_attributes_count?: number;
 };