Tempo: TraceQL table and editor (#59313)

* Tempo: Add the ability to show/hide the Span column in the table when using TraceQL

* Add optional chaining

* Update tests

* Show subcols in a subtable

* Add more space for the subtable

* Remove unused import

* Better expander icon. Improved the subtable styling. Integrated with real data

* Fix expanding the wrong index when table already has an expanded row

* ⚠️ Hack ⚠️ - Fix table links

* Link to spans

* Tempo: [TraceQL] Don't wrap the autocomplete vals for the status tag in quotes

* TraceQL result table improvements and fixes

* Include span name in the subtable

* Loop through data only if it is not nullish

* Integrate traceql with sub-tables

* Added booleans as keywords. Make query editor multiline

* Make date format consistent between trace and span

* Reset expanded indexes when data or subdata change

* Dynamic attributes by trace

* Fix test. Cleanup and refactor

* Tiny refactor

Co-authored-by: Hamas Shafiq <hamas.shafiq@grafana.com>
Co-authored-by: Andrej Ocenas <mr.ocenas@gmail.com>

packages/grafana-ui/src/components/Table/Table.tsx

@@ -329,6 +329,10 @@ export const Table = memo((props: Props) => {
     }
   });
 
+  useEffect(() => {
+    setExpandedIndexes(new Set());
+  }, [data, subData]);
+
   const renderSubTable = React.useCallback(
     (rowIndex: number) => {
       if (expandedIndexes.has(rowIndex)) {

public/app/plugins/datasource/tempo/datasource.ts

@@ -189,7 +189,7 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
           }).pipe(
             map((response) => {
               return {
-                data: [createTableFrameFromTraceQlQuery(response.data.traces, this.instanceSettings)],
+                data: createTableFrameFromTraceQlQuery(response.data.traces, this.instanceSettings),
               };
             }),
             catchError((error) => {

public/app/plugins/datasource/tempo/resultTransformer.test.ts

@@ -117,47 +117,34 @@ describe('createTableFrameFromSearch()', () => {
     expect(frame.fields[2].values.get(0)).toBe('2022-01-28 03:00:28');
     expect(frame.fields[2].values.get(1)).toBe('2022-01-27 22:56:06');
 
-    expect(frame.fields[3].name).toBe('duration');
+    expect(frame.fields[3].name).toBe('traceDuration');
     expect(frame.fields[3].values.get(0)).toBe(65);
   });
 });
 
 describe('createTableFrameFromTraceQlQuery()', () => {
   test('transforms TraceQL response to DataFrame', () => {
-    const frame = createTableFrameFromTraceQlQuery(traceQlResponse.traces, defaultSettings);
+    const frameList = createTableFrameFromTraceQlQuery(traceQlResponse.traces, defaultSettings);
+    const frame = frameList[0];
     // Trace ID field
     expect(frame.fields[0].name).toBe('traceID');
     expect(frame.fields[0].values.get(0)).toBe('b1586c3c8c34d');
     expect(frame.fields[0].config.unit).toBe('string');
     expect(frame.fields[0].values).toBeInstanceOf(ArrayVector);
-    // There should be a traceIdHidden field which is hidden
-    expect(frame.fields[1].name).toBe('traceIdHidden');
-    expect(frame.fields[1].config).toBeDefined();
-    expect(frame.fields[1].config.custom).toStrictEqual({ hidden: true });
-    expect(frame.fields[1].values).toBeInstanceOf(ArrayVector);
-    // Span ID field
-    expect(frame.fields[2].name).toBe('spanID');
-    expect(frame.fields[2].config.unit).toBe('string');
-    expect(frame.fields[2].values).toBeInstanceOf(ArrayVector);
     // Trace name field
-    expect(frame.fields[3].name).toBe('traceName');
+    expect(frame.fields[1].name).toBe('traceName');
-    expect(frame.fields[3].type).toBe('string');
+    expect(frame.fields[1].type).toBe('string');
-    expect(frame.fields[3].values.get(0)).toBe('lb HTTP Client');
+    expect(frame.fields[1].values.get(0)).toBe('lb HTTP Client');
-    expect(frame.fields[3].values).toBeInstanceOf(ArrayVector);
+    expect(frame.fields[1].values).toBeInstanceOf(ArrayVector);
     // Start time field
-    expect(frame.fields[4].name).toBe('startTime');
+    expect(frame.fields[2].name).toBe('startTime');
-    expect(frame.fields[4].type).toBe('string');
+    expect(frame.fields[2].type).toBe('string');
-    expect(frame.fields[4].values.get(1)).toBe('2022-10-19 09:03:34');
+    expect(frame.fields[2].values.get(1)).toBe('2022-01-27 22:56:06');
-    expect(frame.fields[4].values).toBeInstanceOf(ArrayVector);
+    expect(frame.fields[2].values).toBeInstanceOf(ArrayVector);
     // Duration field
-    expect(frame.fields[5].name).toBe('duration');
+    expect(frame.fields[3].name).toBe('traceDuration');
-    expect(frame.fields[5].type).toBe('number');
+    expect(frame.fields[3].type).toBe('number');
-    expect(frame.fields[5].values.get(2)).toBe(6686000);
+    expect(frame.fields[3].values.get(2)).toBe(44);
-    // There should be a field for each attribute
-    expect(frame.fields[6].name).toBe('http.method');
-    expect(frame.fields[6].values).toBeInstanceOf(ArrayVector);
-    expect(frame.fields[7].name).toBe('service.name');
-    expect(frame.fields[7].values).toBeInstanceOf(ArrayVector);
   });
 });
 

public/app/plugins/datasource/tempo/resultTransformer.ts

@@ -1,8 +1,6 @@
 import { SpanStatus, SpanStatusCode } from '@opentelemetry/api';
 import { collectorTypes } from '@opentelemetry/exporter-collector';
 import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
-import differenceInHours from 'date-fns/differenceInHours';
-import formatDistance from 'date-fns/formatDistance';
 
 import {
   DataFrame,
@@ -15,6 +13,7 @@ import {
   TraceSpanReference,
   TraceSpanRow,
   dateTimeFormat,
+  FieldDTO,
 } from '@grafana/data';
 
 import { createGraphFrames } from './graphTransform';
@@ -577,7 +576,7 @@ export function createTableFrameFromSearch(data: TraceSearchMetadata[], instance
       },
       { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Trace name' } },
       { name: 'startTime', type: FieldType.string, config: { displayNameFromDS: 'Start time' } },
-      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ns' } },
+      { name: 'traceDuration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ms' } },
     ],
     meta: {
       preferredVisualisationType: 'table',
@@ -613,8 +612,8 @@ function transformToTraceData(data: TraceSearchMetadata) {
 
   return {
     traceID: data.traceID,
-    startTime: startTime,
+    startTime,
-    duration: data.durationMs?.toString(),
+    traceDuration: data.durationMs?.toString(),
     traceName,
   };
 }
@@ -622,7 +621,7 @@ function transformToTraceData(data: TraceSearchMetadata) {
 export function createTableFrameFromTraceQlQuery(
   data: TraceSearchMetadata[],
   instanceSettings: DataSourceInstanceSettings
-) {
+): DataFrame[] {
   const frame = new MutableDataFrame({
     fields: [
       {
@@ -647,6 +646,58 @@ export function createTableFrameFromTraceQlQuery(
           ],
         },
       },
+      { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Name' } },
+      { name: 'startTime', type: FieldType.string, config: { displayNameFromDS: 'Start time' } },
+      { name: 'traceDuration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ms' } },
+    ],
+    meta: {
+      preferredVisualisationType: 'table',
+    },
+  });
+
+  if (!data?.length) {
+    return [frame];
+  }
+
+  const subDataFrames: DataFrame[] = [];
+  const tableRows = data
+    // Show the most recent traces
+    .sort((a, b) => parseInt(b?.startTimeUnixNano!, 10) / 1000000 - parseInt(a?.startTimeUnixNano!, 10) / 1000000)
+    .reduce((rows: TraceTableData[], trace, currentIndex) => {
+      const traceData: TraceTableData = transformToTraceData(trace);
+      rows.push(traceData);
+      subDataFrames.push(traceSubFrame(trace, instanceSettings, currentIndex));
+      return rows;
+    }, []);
+
+  for (const row of tableRows) {
+    frame.add(row);
+  }
+
+  return [frame, ...subDataFrames];
+}
+
+const traceSubFrame = (
+  trace: TraceSearchMetadata,
+  instanceSettings: DataSourceInstanceSettings,
+  currentIndex: number
+): DataFrame => {
+  const spanDynamicAttrs: Record<string, FieldDTO> = {};
+  let hasNameAttribute = false;
+  trace.spanSet?.spans.forEach((span) => {
+    if (span.name) {
+      hasNameAttribute = true;
+    }
+    span.attributes?.forEach((attr) => {
+      spanDynamicAttrs[attr.key] = {
+        name: attr.key,
+        type: FieldType.string,
+        config: { displayNameFromDS: attr.key },
+      };
+    });
+  });
+  const subFrame = new MutableDataFrame({
+    fields: [
       {
         name: 'traceIdHidden',
         config: {
@@ -670,85 +721,82 @@ export function createTableFrameFromTraceQlQuery(
                   query: '${__data.fields.traceIdHidden}',
                   queryType: 'traceId',
                 },
+                panelsState: {
+                  trace: {
+                    spanId: '${__value.raw}',
+                  },
+                },
               },
             },
           ],
         },
       },
-      { name: 'traceName', type: FieldType.string, config: { displayNameFromDS: 'Trace name' } },
+      {
-      // { name: 'attributes', type: FieldType.string, config: { displayNameFromDS: 'Attributes' } },
+        name: 'name',
-      { name: 'startTime', type: FieldType.string, config: { displayNameFromDS: 'Start time' } },
+        type: FieldType.string,
-      { name: 'duration', type: FieldType.number, config: { displayNameFromDS: 'Duration', unit: 'ns' } },
+        config: { displayNameFromDS: 'Name', custom: { hidden: !hasNameAttribute } },
+      },
+      {
+        name: 'spanStartTime',
+        type: FieldType.string,
+        config: { displayNameFromDS: 'Start time' },
+      },
+      ...Object.values(spanDynamicAttrs),
+      {
+        name: 'duration',
+        type: FieldType.number,
+        config: { displayNameFromDS: 'Duration', unit: 'ns' },
+      },
     ],
     meta: {
       preferredVisualisationType: 'table',
+      custom: {
+        parentRowIndex: currentIndex,
+      },
     },
   });
-  if (!data?.length) {
-    return frame;
-  }
 
-  const attributesAdded: string[] = [];
-
-  data.forEach((trace) => {
   trace.spanSet?.spans.forEach((span) => {
-      span.attributes?.forEach((attr) => {
+    subFrame.add(transformSpanToTraceData(span, trace.traceID));
-        if (!attributesAdded.includes(attr.key)) {
-          frame.addField({ name: attr.key, type: FieldType.string, config: { displayNameFromDS: attr.key } });
-          attributesAdded.push(attr.key);
-        }
-      });
-    });
   });
 
-  const tableRows = data
+  return subFrame;
-    // Show the most recent traces
+};
-    .sort((a, b) => parseInt(b?.startTimeUnixNano!, 10) / 1000000 - parseInt(a?.startTimeUnixNano!, 10) / 1000000)
-    .reduce((rows: TraceTableData[], trace) => {
-      const traceData: TraceTableData = transformToTraceData(trace);
-      rows.push(traceData);
-      trace.spanSet?.spans.forEach((span) => {
-        rows.push(transformSpanToTraceData(span, trace.traceID));
-      });
-      return rows;
-    }, []);
-
-  for (const row of tableRows) {
-    frame.add(row);
-  }
-
-  return frame;
-}
 
 interface TraceTableData {
-  [key: string]: string | number | undefined; // dynamic attribute name
+  [key: string]: string | number | boolean | undefined; // dynamic attribute name
   traceID?: string;
   spanID?: string;
-  //attributes?: string;
   startTime?: string;
-  duration?: string;
+  name?: string;
+  traceDuration?: string;
 }
 
 function transformSpanToTraceData(span: Span, traceID: string): TraceTableData {
   const spanStartTimeUnixMs = parseInt(span.startTimeUnixNano, 10) / 1000000;
   let spanStartTime = dateTimeFormat(spanStartTimeUnixMs);
 
-  if (Math.abs(differenceInHours(new Date(spanStartTime), Date.now())) <= 1) {
-    spanStartTime = formatDistance(new Date(spanStartTime), Date.now(), {
-      addSuffix: true,
-      includeSeconds: true,
-    });
-  }
-
   const data: TraceTableData = {
     traceIdHidden: traceID,
     spanID: span.spanID,
-    startTime: spanStartTime,
+    spanStartTime,
-    duration: span.durationNanos,
+    duration: parseInt(span.durationNanos, 10),
+    name: span.name,
   };
 
   span.attributes?.forEach((attr) => {
+    if (attr.value.boolValue) {
+      data[attr.key] = attr.value.boolValue;
+    }
+    if (attr.value.doubleValue) {
+      data[attr.key] = attr.value.doubleValue;
+    }
+    if (attr.value.intValue) {
+      data[attr.key] = attr.value.intValue;
+    }
+    if (attr.value.stringValue) {
       data[attr.key] = attr.value.stringValue;
+    }
   });
 
   return data;

public/app/plugins/datasource/tempo/testResponse.ts

@@ -2210,6 +2210,8 @@ export const traceQlResponse = {
       traceID: 'b1586c3c8c34d',
       rootServiceName: 'lb',
       rootTraceName: 'HTTP Client',
+      startTimeUnixNano: '1643356828724000000',
+      durationMs: 65,
       spanSet: {
         spans: [
           {
@@ -2296,6 +2298,8 @@ export const traceQlResponse = {
       traceID: '9161e77388f3e',
       rootServiceName: 'lb',
       rootTraceName: 'HTTP Client',
+      startTimeUnixNano: '1643342166678000000',
+      durationMs: 93,
       spanSet: {
         spans: [
           {
@@ -2382,6 +2386,8 @@ export const traceQlResponse = {
       traceID: '480691f7c6f20',
       rootServiceName: 'lb',
       rootTraceName: 'HTTP Client',
+      startTimeUnixNano: '1643342166678000000',
+      durationMs: 44,
       spanSet: {
         spans: [
           {

public/app/plugins/datasource/tempo/traceql/TraceQLEditor.tsx

@@ -33,7 +33,6 @@ export function TraceQLEditor(props: Props) {
       language={langId}
       onBlur={onChange}
       onChange={onChange}
-      height={'30px'}
       containerStyles={styles.queryField}
       monacoOptions={{
         folding: false,
@@ -55,6 +54,7 @@ export function TraceQLEditor(props: Props) {
         setupAutocompleteFn(editor, monaco);
         setupActions(editor, monaco, onRunQuery);
         setupPlaceholder(editor, monaco, styles);
+        setupAutoSize(editor);
       }}
     />
   );
@@ -92,19 +92,30 @@ function setupActions(editor: monacoTypes.editor.IStandaloneCodeEditor, monaco:
   editor.addAction({
     id: 'run-query',
     label: 'Run Query',
-
     keybindings: [monaco.KeyMod.Shift | monaco.KeyCode.Enter],
-
     contextMenuGroupId: 'navigation',
-
     contextMenuOrder: 1.5,
-
     run: function () {
       onRunQuery();
     },
   });
 }
 
+function setupAutoSize(editor: monacoTypes.editor.IStandaloneCodeEditor) {
+  const container = editor.getDomNode();
+  const updateHeight = () => {
+    if (container) {
+      const contentHeight = Math.min(1000, editor.getContentHeight());
+      const width = parseInt(container.style.width, 10);
+      container.style.width = `${width}px`;
+      container.style.height = `${contentHeight}px`;
+      editor.layout({ width, height: contentHeight });
+    }
+  };
+  editor.onDidContentSizeChange(updateHeight);
+  updateHeight();
+}
+
 /**
  * Hook that returns function that will set up monaco autocomplete for the label selector
  * @param datasource

public/app/plugins/datasource/tempo/traceql/autocomplete.ts

@@ -128,13 +128,22 @@ export class CompletionProvider implements monacoTypes.languages.CompletionItemP
         }));
       case 'SPANSET_IN_VALUE':
         const tagName = this.overrideTagName(situation.tagName);
+        const tagsNoQuotesAroundValue: string[] = ['status'];
         const tagValues = await this.getTagValues(tagName);
         const items: Completion[] = [];
+
+        const getInsertionText = (val: SelectableValue<string>): string => {
+          if (situation.betweenQuotes) {
+            return val.label!;
+          }
+          return tagsNoQuotesAroundValue.includes(situation.tagName) ? val.label! : `"${val.label}"`;
+        };
+
         tagValues.forEach((val) => {
           if (val?.label) {
             items.push({
               label: val.label,
-              insertText: situation.betweenQuotes ? val.label : `"${val.label}"`,
+              insertText: getInsertionText(val),
               type: 'TAG_VALUE',
             });
           }

public/app/plugins/datasource/tempo/traceql/traceql.ts

@@ -26,7 +26,9 @@ const intrinsics = ['duration', 'name', 'status', 'parent'];
 
 const scopes: string[] = ['resource', 'span'];
 
-const keywords = intrinsics.concat(scopes);
+const booleans = ['false', 'true'];
+
+const keywords = intrinsics.concat(scopes).concat(booleans);
 
 export const language = {
   ignoreCase: false,

public/app/plugins/datasource/tempo/types.ts

@@ -95,7 +95,10 @@ export type Span = {
   kind?: SpanKind;
   startTimeUnixNano: string;
   endTimeUnixNano?: string;
-  attributes?: Array<{ key: string; value: { stringValue: string } }>;
+  attributes?: Array<{
+    key: string;
+    value: { stringValue?: string; intValue?: string; boolValue?: boolean; doubleValue?: string };
+  }>;
   dropped_attributes_count?: number;
 };