Tempo: Support new TraceQL scopes (#94858)

* Support new TraceQL scopes

* Update TraceQL lezer version

packages/grafana-schema/src/raw/composable/tempo/dataquery/x/TempoDataQuery_types.gen.ts

@@ -98,7 +98,10 @@ export enum SearchTableType {
  * static fields are pre-set in the UI, dynamic fields are added by the user
  */
 export enum TraceqlSearchScope {
+  Event = 'event',
+  Instrumentation = 'instrumentation',
   Intrinsic = 'intrinsic',
+  Link = 'link',
   Resource = 'resource',
   Span = 'span',
   Unscoped = 'unscoped',

pkg/tsdb/tempo/kinds/dataquery/types_dataquery_gen.go

@@ -37,10 +37,13 @@ const (
 
 // Defines values for TraceqlSearchScope.
 const (
-	TraceqlSearchScopeIntrinsic TraceqlSearchScope = "intrinsic"
+	TraceqlSearchScopeEvent           TraceqlSearchScope = "event"
-	TraceqlSearchScopeResource  TraceqlSearchScope = "resource"
+	TraceqlSearchScopeInstrumentation TraceqlSearchScope = "instrumentation"
-	TraceqlSearchScopeSpan      TraceqlSearchScope = "span"
+	TraceqlSearchScopeIntrinsic       TraceqlSearchScope = "intrinsic"
-	TraceqlSearchScopeUnscoped  TraceqlSearchScope = "unscoped"
+	TraceqlSearchScopeLink            TraceqlSearchScope = "link"
+	TraceqlSearchScopeResource        TraceqlSearchScope = "resource"
+	TraceqlSearchScopeSpan            TraceqlSearchScope = "span"
+	TraceqlSearchScopeUnscoped        TraceqlSearchScope = "unscoped"
 )
 
 // These are the common properties available to all queries in all datasources.

public/app/plugins/datasource/tempo/SearchTraceQLEditor/GroupByField.test.tsx

@@ -35,6 +35,7 @@ describe('GroupByField', () => {
   };
 
   jest.spyOn(lp, 'getMetricsSummaryTags').mockReturnValue(['component', 'http.method', 'http.status_code']);
+  jest.spyOn(lp, 'getTags').mockReturnValue(['component', 'http.method', 'http.status_code']);
 
   beforeEach(() => {
     jest.useFakeTimers();

public/app/plugins/datasource/tempo/SearchTraceQLEditor/GroupByField.tsx

@@ -78,14 +78,19 @@ export const GroupByField = (props: Props) => {
     onChange(copy);
   };
 
-  const scopeOptions = Object.values(TraceqlSearchScope).map((t) => ({ label: t, value: t }));
+  const scopeOptions = Object.values(TraceqlSearchScope)
+    .filter((s) => {
+      // only add scope if it has tags
+      return datasource.languageProvider.getTags(s).length > 0;
+    })
+    .map((t) => ({ label: t, value: t }));
 
   return (
     <InlineSearchField label="Aggregate by" tooltip={`${notice} Select one or more tags to see the metrics summary.`}>
       <>
         {query.groupBy?.map((f, i) => {
           const tags = tagOptions(f)
-            ?.concat(f.tag !== undefined && !tagOptions(f)?.includes(f.tag) ? [f.tag] : [])
+            ?.concat(f.tag !== undefined && f.tag !== '' && !tagOptions(f)?.includes(f.tag) ? [f.tag] : [])
             .map((t) => ({
               label: t,
               value: t,

public/app/plugins/datasource/tempo/SearchTraceQLEditor/SearchField.test.tsx

@@ -153,7 +153,7 @@ describe('SearchField', () => {
     }
   });
 
-  it('should not provide intrinsic as a selectable scope', async () => {
+  it('should provide intrinsic as a selectable scope', async () => {
     const updateFilter = jest.fn((val) => {
       return val;
     });
@@ -171,7 +171,7 @@ describe('SearchField', () => {
       expect(await screen.findByText('resource')).toBeInTheDocument();
       expect(await screen.findByText('span')).toBeInTheDocument();
       expect(await screen.findByText('unscoped')).toBeInTheDocument();
-      expect(screen.queryByText('intrinsic')).not.toBeInTheDocument();
+      expect(await screen.findByText('intrinsic')).toBeInTheDocument();
       expect(await screen.findByText('$templateVariable1')).toBeInTheDocument();
       expect(await screen.findByText('$templateVariable2')).toBeInTheDocument();
     }
@@ -188,6 +188,7 @@ describe('SearchField', () => {
         },
       ]),
       getIntrinsics: jest.fn().mockReturnValue(['duration']),
+      getTags: jest.fn().mockReturnValue(['cluster']),
     } as unknown as TempoLanguageProvider;
 
     const { container } = renderSearchField(jest.fn(), filter, [], false, lp);
@@ -237,6 +238,7 @@ describe('SearchField', () => {
         },
       ]),
       getIntrinsics: jest.fn().mockReturnValue(['duration']),
+      getTags: jest.fn().mockReturnValue(['cluster']),
     } as unknown as TempoLanguageProvider;
 
     const { container } = renderSearchField(jest.fn(), filter, [], false, lp);
@@ -283,6 +285,7 @@ const renderSearchField = (
         },
       ]),
       getIntrinsics: jest.fn().mockReturnValue(['duration']),
+      getTags: jest.fn().mockReturnValue(['cluster']),
     } as unknown as TempoLanguageProvider);
 
   const datasource: TempoDatasource = {

public/app/plugins/datasource/tempo/SearchTraceQLEditor/SearchField.tsx

@@ -114,7 +114,10 @@ const SearchField = ({
   }, [filter.value]);
 
   const scopeOptions = Object.values(TraceqlSearchScope)
-    .filter((s) => s !== TraceqlSearchScope.Intrinsic)
+    .filter((s) => {
+      // only add scope if it has tags
+      return datasource.languageProvider.getTags(s).length > 0;
+    })
     .map((t) => ({ label: t, value: t }));
 
   // If all values have type string or int/float use a focused list of operators instead of all operators
@@ -177,7 +180,7 @@ const SearchField = ({
             inputId={`${filter.id}-scope`}
             options={addVariablesToOptions ? withTemplateVariableOptions(scopeOptions) : scopeOptions}
             value={filter.scope}
-            onChange={(v) => updateFilter({ ...filter, scope: v?.value })}
+            onChange={(v) => updateFilter({ ...filter, scope: v?.value, tag: undefined, value: [] })}
             placeholder="Select scope"
             aria-label={`select ${filter.id} scope`}
           />
@@ -197,6 +200,7 @@ const SearchField = ({
             onCloseMenu={() => setTagQuery('')}
             onChange={(v) => updateFilter({ ...filter, tag: v?.value, value: [] })}
             value={filter.tag}
+            key={filter.tag}
             placeholder="Select tag"
             isClearable
             aria-label={`select ${filter.id} tag`}

public/app/plugins/datasource/tempo/SearchTraceQLEditor/TagsInput.test.tsx

@@ -43,7 +43,6 @@ describe('TagsInput', () => {
         jest.advanceTimersByTime(1000);
       });
       await waitFor(() => {
-        expect(screen.getByText('rootServiceName')).toBeInTheDocument();
         expect(screen.getByText('bar')).toBeInTheDocument();
       });
     });

public/app/plugins/datasource/tempo/SearchTraceQLEditor/TagsInput.tsx

@@ -69,7 +69,7 @@ const TagsInput = ({
 
   const getTags = (f: TraceqlFilter) => {
     const tags = datasource.languageProvider.getTags(f.scope);
-    return getFilteredTags(tags, datasource.languageProvider, staticTags);
+    return getFilteredTags(tags, staticTags);
   };
 
   const validInput = (f: TraceqlFilter) => {

public/app/plugins/datasource/tempo/SearchTraceQLEditor/utils.test.ts

@@ -3,7 +3,6 @@ import { uniq } from 'lodash';
 import { TraceqlSearchScope } from '../dataquery.gen';
 import { TempoDatasource } from '../datasource';
 import TempoLanguageProvider from '../language_provider';
-import { intrinsicsV1 } from '../traceql/traceql';
 
 import { getUnscopedTags, getFilteredTags, getAllTags, getTagsByScope, generateQueryFromAdHocFilters } from './utils';
 
@@ -51,34 +50,34 @@ describe('gets correct tags', () => {
   const lp = new TempoLanguageProvider(datasource);
 
   it('for filtered tags when no tags supplied', () => {
-    const tags = getFilteredTags(emptyTags, lp, []);
+    const tags = getFilteredTags(emptyTags, []);
-    expect(tags).toEqual(intrinsicsV1);
+    expect(tags).toEqual([]);
   });
 
   it('for filtered tags when API v1 tags supplied', () => {
-    const tags = getFilteredTags(v1Tags, lp, []);
+    const tags = getFilteredTags(v1Tags, []);
-    expect(tags).toEqual(intrinsicsV1.concat(['bar', 'foo']));
+    expect(tags).toEqual(['bar', 'foo']);
   });
 
   it('for filtered tags when API v1 tags supplied with tags to filter out', () => {
-    const tags = getFilteredTags(v1Tags, lp, ['duration']);
+    const tags = getFilteredTags(v1Tags, ['foo']);
-    expect(tags).toEqual(intrinsicsV1.filter((x) => x !== 'duration').concat(['bar', 'foo']));
+    expect(tags).toEqual(['bar']);
   });
 
   it('for filtered tags when API v2 tags supplied', () => {
-    const tags = getFilteredTags(uniq(getUnscopedTags(v2Tags)), lp, []);
+    const tags = getFilteredTags(uniq(getUnscopedTags(v2Tags)), []);
-    expect(tags).toEqual(intrinsicsV1.concat(['cluster', 'container', 'db']));
+    expect(tags).toEqual(['cluster', 'container', 'db']);
   });
 
   it('for filtered tags when API v2 tags supplied with tags to filter out', () => {
-    const tags = getFilteredTags(getUnscopedTags(v2Tags), lp, ['duration', 'cluster']);
+    const tags = getFilteredTags(getUnscopedTags(v2Tags), ['cluster']);
-    expect(tags).toEqual(intrinsicsV1.filter((x) => x !== 'duration').concat(['container', 'db']));
+    expect(tags).toEqual(['container', 'db']);
   });
 
   it('for filtered tags when API v2 tags set', () => {
     lp.setV2Tags(v2Tags);
-    const tags = getFilteredTags(uniq(getUnscopedTags(v2Tags)), lp, []);
+    const tags = getFilteredTags(uniq(getUnscopedTags(v2Tags)), []);
-    expect(tags).toEqual(testIntrinsics.concat(['cluster', 'container', 'db']));
+    expect(tags).toEqual(['cluster', 'container', 'db']);
   });
 
   it('for unscoped tags', () => {

public/app/plugins/datasource/tempo/SearchTraceQLEditor/utils.ts

@@ -44,7 +44,13 @@ export const scopeHelper = (f: TraceqlFilter, lp: TempoLanguageProvider) => {
     return '';
   }
   return (
-    (f.scope === TraceqlSearchScope.Resource || f.scope === TraceqlSearchScope.Span ? f.scope?.toLowerCase() : '') + '.'
+    (f.scope === TraceqlSearchScope.Event ||
+    f.scope === TraceqlSearchScope.Instrumentation ||
+    f.scope === TraceqlSearchScope.Link ||
+    f.scope === TraceqlSearchScope.Resource ||
+    f.scope === TraceqlSearchScope.Span
+      ? f.scope?.toLowerCase()
+      : '') + '.'
   );
 };
 
@@ -77,7 +83,7 @@ const adHocValueHelper = (f: AdHocVariableFilter, lp: TempoLanguageProvider) =>
 };
 
 export const getTagWithoutScope = (tag: string) => {
-  return tag.replace(/^(event|link|resource|span)\./, '');
+  return tag.replace(/^(event|instrumentation|link|resource|span)\./, '');
 };
 
 export const filterScopedTag = (f: TraceqlFilter, lp: TempoLanguageProvider) => {
@@ -96,12 +102,8 @@ export const filterTitle = (f: TraceqlFilter, lp: TempoLanguageProvider) => {
   return startCase(filterScopedTag(f, lp));
 };
 
-export const getFilteredTags = (
+export const getFilteredTags = (tags: string[], staticTags: Array<string | undefined>) => {
-  tags: string[],
+  return [...tags].filter((t) => !staticTags.includes(t));
-  languageProvider: TempoLanguageProvider,
-  staticTags: Array<string | undefined>
-) => {
-  return [...languageProvider.getIntrinsics(), ...tags].filter((t) => !staticTags.includes(t));
 };
 
 export const getUnscopedTags = (scopes: Scope[]) => {

public/app/plugins/datasource/tempo/dataquery.cue

@@ -64,7 +64,7 @@ composableKinds: DataQuery: {
 				#SearchTableType: "traces" | "spans" | "raw" @cuetsy(kind="enum")
 
 				// static fields are pre-set in the UI, dynamic fields are added by the user
-				#TraceqlSearchScope: "intrinsic" | "unscoped" | "resource" | "span" @cuetsy(kind="enum")
+				#TraceqlSearchScope: "intrinsic" | "unscoped" | "event" | "instrumentation" | "link" | "resource" | "span" @cuetsy(kind="enum")
 				#TraceqlFilter: {
 					// Uniquely identify the filter, will not be used in the query generation
 					id: string

public/app/plugins/datasource/tempo/dataquery.gen.ts

@@ -96,7 +96,10 @@ export enum SearchTableType {
  * static fields are pre-set in the UI, dynamic fields are added by the user
  */
 export enum TraceqlSearchScope {
+  Event = 'event',
+  Instrumentation = 'instrumentation',
   Intrinsic = 'intrinsic',
+  Link = 'link',
   Resource = 'resource',
   Span = 'span',
   Unscoped = 'unscoped',

public/app/plugins/datasource/tempo/traceql/autocomplete.ts

@@ -528,7 +528,7 @@ function fixSuggestion(
     const match = model
       .getValue()
       .substring(0, offset)
-      .match(/(span\.|resource\.|\.)?([\w./-]*)$/);
+      .match(/(event\.|instrumentation\.|link\.|resource\.|span\.|\.)?([\w./-]*)$/);
 
     if (match) {
       const scope = match[1];

public/app/plugins/datasource/tempo/traceql/highlighting.ts

@@ -5,11 +5,14 @@ import {
   And,
   AttributeField,
   ComparisonOp,
+  Event,
   FieldExpression,
   FieldOp,
   GroupOperation,
   Identifier,
+  Instrumentation,
   IntrinsicField,
+  Link,
   Or,
   Parent,
   parser,
@@ -157,6 +160,9 @@ export const getWarningMarkers = (severity: number, model: monacoTypes.editor.IT
         // Make sure prevSibling is using the proper scope
         if (
           node.prevSibling?.type.id !== Parent &&
+          node.prevSibling?.type.id !== Event &&
+          node.prevSibling?.type.id !== Instrumentation &&
+          node.prevSibling?.type.id !== Link &&
           node.prevSibling?.type.id !== Resource &&
           node.prevSibling?.type.id !== Span
         ) {

public/app/plugins/datasource/tempo/traceql/situation.ts

@@ -340,7 +340,9 @@ function resolveAttribute(node: SyntaxNode, text: string): SituationType {
   const indexOfDot = attributeFieldParentText.indexOf('.');
   const attributeFieldUpToDot = attributeFieldParentText.slice(0, indexOfDot);
 
-  if (['span', 'resource', 'parent'].find((item) => item === attributeFieldUpToDot)) {
+  if (
+    ['event', 'instrumentation', 'link', 'resource', 'span', 'parent'].find((item) => item === attributeFieldUpToDot)
+  ) {
     return {
       type: 'SPANSET_IN_NAME_SCOPE',
       scope: attributeFieldUpToDot,

public/app/plugins/datasource/tempo/traceql/traceql.ts

@@ -56,7 +56,7 @@ export const intrinsics = intrinsicsV1.concat([
   'trace:rootName',
   'trace:rootService',
 ]);
-export const scopes: string[] = ['resource', 'span'];
+export const scopes: string[] = ['event', 'instrumentation', 'link', 'resource', 'span'];
 
 const aggregatorFunctions = ['avg', 'count', 'max', 'min', 'sum'];
 const functions = aggregatorFunctions.concat([
@@ -198,13 +198,14 @@ export const traceqlGrammar: Grammar = {
     pattern: /\{[^}]*}/,
     inside: {
       filter: {
-        pattern: /([\w.\/-]+)?(\s*)(([!=+\-<>~]+)\s*("([^"\n&]+)?"?|([^"\n\s&|}]+))?)/g,
+        pattern:
+          /([\w:.\/-]+)\s*(=|!=|<=|>=|=~|!~|>|<)\s*("[^"]*"|[\w.\/-]+)(\s*(\&\&|\|\|)\s*([\w:.\/-]+)\s*(=|!=|<=|>=|=~|!~|>|<)\s*("[^"]*"|[\w.\/-]+))*/g,
         inside: {
           comment: {
             pattern: /#.*/,
           },
           'label-key': {
-            pattern: /[a-z_.][\w./_-]*(?=\s*(=|!=|>|<|>=|<=|=~|!~))/,
+            pattern: /[a-z_.][\w./_-]*(:[\w./_-]+)?(?=\s*(=|!=|>|<|>=|<=|=~|!~))/,
             alias: 'attr-name',
           },
           'label-value': {