Tempo: Integrate TraceQL grammar (#72516)

* Replacing the regex based approach and integrating the lezer-traceql grammar in the editor

* Added some tests

* Added path to capture error in filter root

* Tests are great and help fix issues 👍

* Fix autocomplete tests
This commit is contained in:
Andre Pereira 2023-08-01 14:15:05 +01:00 committed by GitHub
parent 2bfef9e916
commit 5c14a07ec3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 314 additions and 178 deletions

View File

@ -269,6 +269,7 @@
"@grafana/faro-web-sdk": "1.1.2",
"@grafana/google-sdk": "0.1.1",
"@grafana/lezer-logql": "0.1.8",
"@grafana/lezer-traceql": "0.0.4",
"@grafana/monaco-logql": "^0.0.7",
"@grafana/runtime": "workspace:*",
"@grafana/scenes": "0.22.0",

View File

@ -45,7 +45,7 @@ describe('CompletionProvider', () => {
});
it('does not wrap the tag value in quotes if the type in the response is something other than "string"', async () => {
const { provider, model } = setup('{foo=}', 5, v1Tags);
const { provider, model } = setup('{.foo=}', 6, v1Tags);
jest.spyOn(provider.languageProvider, 'getOptionsV2').mockImplementation(
() =>
@ -70,7 +70,7 @@ describe('CompletionProvider', () => {
});
it('wraps the tag value in quotes if the type in the response is set to "string"', async () => {
const { provider, model } = setup('{foo=}', 5, v1Tags);
const { provider, model } = setup('{.foo=}', 6, v1Tags);
jest.spyOn(provider.languageProvider, 'getOptionsV2').mockImplementation(
() =>
@ -95,7 +95,7 @@ describe('CompletionProvider', () => {
});
it('inserts the tag value without quotes if the user has entered quotes', async () => {
const { provider, model } = setup('{foo="}', 6, v1Tags);
const { provider, model } = setup('{.foo="}', 6, v1Tags);
jest.spyOn(provider.languageProvider, 'getOptionsV2').mockImplementation(
() =>
@ -119,7 +119,7 @@ describe('CompletionProvider', () => {
});
it('suggests nothing without tags', async () => {
const { provider, model } = setup('{foo="}', 7, emptyTags);
const { provider, model } = setup('{.foo="}', 8, emptyTags);
const result = await provider.provideCompletionItems(
model as unknown as monacoTypes.editor.ITextModel,
{} as monacoTypes.Position
@ -180,13 +180,15 @@ describe('CompletionProvider', () => {
});
it('suggests operators after a space after the tag name', async () => {
const { provider, model } = setup('{ foo }', 6, v1Tags);
const { provider, model } = setup('{ .foo }', 7, v1Tags);
const result = await provider.provideCompletionItems(
model as unknown as monacoTypes.editor.ITextModel,
{} as monacoTypes.Position
);
expect((result! as monacoTypes.languages.CompletionList).suggestions).toEqual(
CompletionProvider.operators.map((s) => expect.objectContaining({ label: s, insertText: s }))
[...CompletionProvider.logicalOps, ...CompletionProvider.operators].map((s) =>
expect.objectContaining({ label: s, insertText: s })
)
);
});
@ -224,38 +226,16 @@ describe('CompletionProvider', () => {
});
it('suggests logical operators and close bracket after the value', async () => {
const { provider, model } = setup('{foo=300 }', 9, v1Tags);
const { provider, model } = setup('{.foo=300 }', 10, v1Tags);
const result = await provider.provideCompletionItems(
model as unknown as monacoTypes.editor.ITextModel,
{} as monacoTypes.Position
);
expect((result! as monacoTypes.languages.CompletionList).suggestions).toEqual([
...CompletionProvider.logicalOps.map((s) => expect.objectContaining({ label: s, insertText: s })),
expect.objectContaining({ label: '}', insertText: '}' }),
]);
});
it('suggests tag values after a space inside a string', async () => {
const { provider, model } = setup('{foo="bar test " }', 15, v1Tags);
jest.spyOn(provider.languageProvider, 'getOptionsV2').mockImplementation(
() =>
new Promise((resolve) => {
resolve([
{
value: 'foobar',
label: 'foobar',
},
]);
})
expect((result! as monacoTypes.languages.CompletionList).suggestions).toEqual(
[...CompletionProvider.logicalOps, ...CompletionProvider.operators].map((s) =>
expect.objectContaining({ label: s, insertText: s })
)
);
const result = await provider.provideCompletionItems(
model as unknown as monacoTypes.editor.ITextModel,
{} as monacoTypes.Position
);
expect((result! as monacoTypes.languages.CompletionList).suggestions).toEqual([
expect.objectContaining({ label: 'foobar', insertText: 'foobar' }),
]);
});
});

View File

@ -7,6 +7,7 @@ import { notifyApp } from '../../../../core/reducers/appNotification';
import { dispatch } from '../../../../store/store';
import TempoLanguageProvider from '../language_provider';
import { getSituation, Situation } from './situation';
import { intrinsics, scopes } from './traceql';
interface Props {
@ -52,8 +53,8 @@ export class CompletionProvider implements monacoTypes.languages.CompletionItemP
}
const { range, offset } = getRangeAndOffset(this.monaco, model, position);
const situation = this.getSituation(model.getValue(), offset);
const completionItems = this.getCompletions(situation);
const situation = getSituation(model.getValue(), offset);
const completionItems = situation != null ? this.getCompletions(situation) : Promise.resolve([]);
return completionItems.then((items) => {
// monaco by-default alphabetically orders the items.
@ -124,8 +125,8 @@ export class CompletionProvider implements monacoTypes.languages.CompletionItemP
return this.getScopesCompletions().concat(this.getIntrinsicsCompletions()).concat(this.getTagsCompletions());
case 'SPANSET_IN_NAME_SCOPE':
return this.getTagsCompletions(undefined, situation.scope);
case 'SPANSET_AFTER_NAME':
return CompletionProvider.operators.map((key) => ({
case 'SPANSET_EXPRESSION_OPERATORS':
return [...CompletionProvider.logicalOps, ...CompletionProvider.operators].map((key) => ({
label: key,
insertText: key,
type: 'OPERATOR',
@ -198,115 +199,6 @@ export class CompletionProvider implements monacoTypes.languages.CompletionItemP
type: 'SCOPE',
}));
}
private getSituationInSpanSet(textUntilCaret: string): Situation {
const nameRegex = /(?<name>[\w./-]+)?/;
const opRegex = /(?<op>[!=+\-<>]+)/;
// only allow spaces in the value if it's enclosed by quotes
const valueRegex = /(?<value>(?<open_quote>")([^"\n&|]+)?(?<close_quote>")?|([^"\n\s&|]+))?/;
// prettier-ignore
const fullRegex = new RegExp(
'([\\s{])' + // Space(s) or initial opening bracket {
'(' + // Open full set group
nameRegex.source +
'(?<space1>\\s*)' + // Optional space(s) between name and operator
'(' + // Open operator + value group
opRegex.source +
'(?<space2>\\s*)' + // Optional space(s) between operator and value
valueRegex.source +
')?' + // Close operator + value group
')' + // Close full set group
'(?<space3>\\s*)$' // Optional space(s) at the end of the set
);
const matched = textUntilCaret.match(fullRegex);
if (matched) {
const nameFull = matched.groups?.name;
const op = matched.groups?.op;
if (!nameFull) {
return {
type: 'SPANSET_EMPTY',
};
}
if (nameFull === '.') {
return {
type: 'SPANSET_ONLY_DOT',
};
}
const nameMatched = nameFull.match(/^(?<pre_dot>\.)?(?<word>\w[\w./-]*\w)(?<post_dot>\.)?$/);
// We already have a (potentially partial) tag name so let's check if there's an operator declared
// { .tag_name|
if (!op) {
// There's no operator so we check if the name is one of the known scopes
// { resource.|
if (scopes.filter((w) => w === nameMatched?.groups?.word) && nameMatched?.groups?.post_dot) {
return {
type: 'SPANSET_IN_NAME_SCOPE',
scope: nameMatched?.groups?.word || '',
};
}
// It's not one of the scopes, so we now check if we're after the name (there's a space after the word) or if we still have to autocomplete the rest of the name
// In case there's a space we start autocompleting the operators { .http.method |
// Otherwise we keep showing the tags/intrinsics/scopes list { .http.met|
return {
type: matched.groups?.space1 ? 'SPANSET_AFTER_NAME' : 'SPANSET_IN_NAME',
};
}
// In case there's a space after the full [name + operator + value] group we can start autocompleting logical operators or close the spanset
// To avoid triggering this situation when we are writing a space inside a string we check the state of the open and close quotes
// { .http.method = "GET" |
if (matched.groups?.space3 && matched.groups.open_quote === matched.groups.close_quote) {
return {
type: 'SPANSET_AFTER_VALUE',
};
}
// We already have an operator and know that the set isn't complete so let's autocomplete the possible values for the tag name
// { .http.method = |
return {
type: 'SPANSET_IN_VALUE',
tagName: nameFull,
betweenQuotes: !!matched.groups?.open_quote,
};
}
return {
type: 'EMPTY',
};
}
/**
* Figure out where is the cursor and what kind of suggestions are appropriate.
* @param text
* @param offset
*/
private getSituation(text: string, offset: number): Situation {
if (text === '' || offset === 0) {
return {
type: 'EMPTY',
};
}
const textUntilCaret = text.substring(0, offset);
// Check if we're inside a span set
let isInSpanSet = textUntilCaret.lastIndexOf('{') > textUntilCaret.lastIndexOf('}');
if (isInSpanSet) {
return this.getSituationInSpanSet(textUntilCaret);
}
// Will happen only if user writes something that isn't really a tag selector
return {
type: 'UNKNOWN',
};
}
}
/**
@ -343,38 +235,6 @@ export type Tag = {
value: string;
};
export type Situation =
| {
type: 'UNKNOWN';
}
| {
type: 'EMPTY';
}
| {
type: 'SPANSET_EMPTY';
}
| {
type: 'SPANSET_ONLY_DOT';
}
| {
type: 'SPANSET_AFTER_NAME';
}
| {
type: 'SPANSET_IN_NAME';
}
| {
type: 'SPANSET_IN_NAME_SCOPE';
scope: string;
}
| {
type: 'SPANSET_IN_VALUE';
tagName: string;
betweenQuotes: boolean;
}
| {
type: 'SPANSET_AFTER_VALUE';
};
function getRangeAndOffset(monaco: Monaco, model: monacoTypes.editor.ITextModel, position: monacoTypes.Position) {
const word = model.getWordAtPosition(position);
const range =

View File

@ -0,0 +1,68 @@
import { getSituation, Situation } from './situation';
jest.mock('@grafana/runtime', () => ({
...jest.requireActual('@grafana/runtime'),
}));
interface SituationTest {
query: string;
cursorPos: number;
expected: Situation;
}
describe('situation', () => {
const tests: SituationTest[] = [
{
query: '{}',
cursorPos: 1,
expected: { type: 'SPANSET_EMPTY' },
},
{
query: '{.}',
cursorPos: 2,
expected: { type: 'SPANSET_ONLY_DOT' },
},
{
query: '{foo}',
cursorPos: 4,
expected: { type: 'SPANSET_IN_NAME' },
},
{
query: '{span.}',
cursorPos: 6,
expected: { type: 'SPANSET_IN_NAME_SCOPE', scope: 'span' },
},
{
query: '{span.foo }',
cursorPos: 10,
expected: { type: 'SPANSET_EXPRESSION_OPERATORS' },
},
{
query: '{span.foo = }',
cursorPos: 12,
expected: { type: 'SPANSET_IN_VALUE', tagName: 'span.foo', betweenQuotes: false },
},
{
query: '{span.foo = "val" }',
cursorPos: 18,
expected: { type: 'SPANSET_EXPRESSION_OPERATORS' },
},
{
query: '{span.foo = "val" && }',
cursorPos: 21,
expected: { type: 'SPANSET_EMPTY' },
},
{
query: '{span.foo = "val" && resource.}',
cursorPos: 30,
expected: { type: 'SPANSET_IN_NAME_SCOPE', scope: 'resource' },
},
];
tests.forEach((test) => {
it(`${test.query} at ${test.cursorPos} is ${test.expected.type}`, async () => {
const sit = getSituation(test.query, test.cursorPos);
expect(sit).toEqual(test.expected);
});
});
});

View File

@ -0,0 +1,217 @@
// we find the first error-node in the tree that is at the cursor-position.
// NOTE: this might be too slow, might need to optimize it
// (ideas: we do not need to go into every subtree, based on from/to)
// also, only go to places that are in the sub-tree of the node found
// by default by lezer. problem is, `next()` will go upward too,
// and we do not want to go higher than our node
import { SyntaxNode, Tree } from '@lezer/common';
import { AttributeField, FieldExpression, FieldOp, parser, SpansetFilter } from '@grafana/lezer-traceql';
type Direction = 'parent' | 'firstChild' | 'lastChild' | 'nextSibling' | 'prevSibling';
type NodeType = number;
export type Situation =
| {
type: 'UNKNOWN';
}
| {
type: 'EMPTY';
}
| {
type: 'SPANSET_EMPTY';
}
| {
type: 'SPANSET_ONLY_DOT';
}
| {
type: 'SPANSET_EXPRESSION_OPERATORS';
}
| {
type: 'SPANSET_IN_NAME';
}
| {
type: 'SPANSET_IN_NAME_SCOPE';
scope: string;
}
| {
type: 'SPANSET_IN_VALUE';
tagName: string;
betweenQuotes: boolean;
}
| {
type: 'SPANSET_AFTER_VALUE';
};
type Path = Array<[Direction, NodeType[]]>;
type Resolver = {
path: NodeType[];
fun: (node: SyntaxNode, text: string, pos: number) => Situation | null;
};
function getErrorNode(tree: Tree, cursorPos: number): SyntaxNode | null {
const cur = tree.cursorAt(cursorPos);
do {
if (cur.from === cursorPos || cur.to === cursorPos) {
const { node } = cur;
if (node.type.isError) {
return node;
}
}
} while (cur.next());
return null;
}
function move(node: SyntaxNode, direction: Direction): SyntaxNode | null {
return node[direction];
}
function walk(node: SyntaxNode, path: Path): SyntaxNode | null {
let current: SyntaxNode | null = node;
for (const [direction, expectedNodes] of path) {
current = move(current, direction);
if (current === null) {
// we could not move in the direction, we stop
return null;
}
if (!expectedNodes.find((en) => en === current?.type.id)) {
// the reached node has wrong type, we stop
return null;
}
}
return current;
}
function getNodeText(node: SyntaxNode, text: string): string {
// if the from and to are them same (e.g. for an error node) we can subtract 1 from the start/from index
return text.slice(node.from === node.to ? node.from - 1 : node.from, node.to);
}
function isPathMatch(resolverPath: NodeType[], cursorPath: number[]): boolean {
return resolverPath.every((item, index) => item === cursorPath[index]);
}
/**
* Figure out where is the cursor and what kind of suggestions are appropriate.
* @param text
* @param offset
*/
export function getSituation(text: string, offset: number): Situation | null {
// there is a special case when we are at the start of writing text,
// so we handle that case first
if (text === '') {
return {
type: 'EMPTY',
};
}
const tree = parser.parse(text);
// if the tree contains error, it is very probable that
// our node is one of those error nodes.
// also, if there are errors, the node lezer finds us,
// might not be the best node.
// so first we check if there is an error node at the cursor position
let maybeErrorNode = getErrorNode(tree, offset);
if (!maybeErrorNode) {
// try again with the previous character
maybeErrorNode = getErrorNode(tree, offset - 1);
}
const cur = maybeErrorNode != null ? maybeErrorNode.cursor() : tree.cursorAt(offset);
const currentNode = cur.node;
const ids = [cur.type.id];
while (cur.parent()) {
ids.push(cur.type.id);
}
for (let resolver of RESOLVERS) {
if (isPathMatch(resolver.path, ids)) {
return resolver.fun(currentNode, text, offset);
}
}
return null;
}
const ERROR_NODE_ID = 0;
const RESOLVERS: Resolver[] = [
{
path: [ERROR_NODE_ID, AttributeField],
fun: resolveAttribute,
},
{
path: [ERROR_NODE_ID, FieldExpression],
fun: resolveExpression,
},
{
path: [ERROR_NODE_ID, SpansetFilter],
fun: resolveErrorInFilterRoot,
},
{
path: [SpansetFilter],
fun: resolveSpanset,
},
];
function resolveSpanset(node: SyntaxNode, text: string, pos: number): Situation {
const lastFieldExpression = walk(node, [['lastChild', [FieldExpression]]]);
if (lastFieldExpression) {
return {
type: 'SPANSET_EXPRESSION_OPERATORS',
};
}
return {
type: 'SPANSET_EMPTY',
};
}
function resolveAttribute(node: SyntaxNode, text: string, pos: number): Situation {
const attributeFieldParent = walk(node, [['parent', [AttributeField]]]);
const attributeFieldParentText = attributeFieldParent ? getNodeText(attributeFieldParent, text) : '';
if (attributeFieldParentText === '.') {
return {
type: 'SPANSET_ONLY_DOT',
};
}
const indexOfDot = attributeFieldParentText.indexOf('.');
const attributeFieldUpToDot = attributeFieldParentText.slice(0, indexOfDot);
if (['span', 'resource', 'parent'].find((item) => item === attributeFieldUpToDot)) {
return {
type: 'SPANSET_IN_NAME_SCOPE',
scope: attributeFieldUpToDot,
};
}
return {
type: 'SPANSET_IN_NAME',
};
}
function resolveExpression(node: SyntaxNode, text: string, pos: number): Situation {
if (node.prevSibling?.type.id === FieldOp) {
let attributeField = node.prevSibling.prevSibling;
if (attributeField) {
return {
type: 'SPANSET_IN_VALUE',
tagName: getNodeText(attributeField, text),
betweenQuotes: false,
};
}
}
return {
type: 'SPANSET_EMPTY',
};
}
function resolveErrorInFilterRoot(node: SyntaxNode, text: string, pos: number): Situation {
return {
type: 'SPANSET_IN_NAME',
};
}

View File

@ -3868,6 +3868,15 @@ __metadata:
languageName: node
linkType: hard
"@grafana/lezer-traceql@npm:0.0.4":
version: 0.0.4
resolution: "@grafana/lezer-traceql@npm:0.0.4"
peerDependencies:
"@lezer/lr": ^1.3.0
checksum: 69acea33476d3cdabfb99f3eb62bb34289bc205da69920e0eccc82f40407f9d584fa03f9662706ab667ee97d3ea84b964b22a11925e6699deef5afe1ca9e1906
languageName: node
linkType: hard
"@grafana/monaco-logql@npm:^0.0.7":
version: 0.0.7
resolution: "@grafana/monaco-logql@npm:0.0.7"
@ -19282,6 +19291,7 @@ __metadata:
"@grafana/faro-web-sdk": 1.1.2
"@grafana/google-sdk": 0.1.1
"@grafana/lezer-logql": 0.1.8
"@grafana/lezer-traceql": 0.0.4
"@grafana/monaco-logql": ^0.0.7
"@grafana/runtime": "workspace:*"
"@grafana/scenes": 0.22.0