diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts index 386f710c155..306dfc50eb3 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts @@ -290,26 +290,58 @@ describe('buildVisualQueryFromString', () => { it('parses binary query', () => { expect(buildVisualQueryFromString('rate({project="bar"}[5m]) / rate({project="foo"}[5m])')).toEqual( noErrors({ - labels: [ - { - op: '=', - value: 'bar', - label: 'project', - }, - ], + labels: [{ op: '=', value: 'bar', label: 'project' }], operations: [{ id: 'rate', params: ['5m'] }], binaryQueries: [ { operator: '/', query: { - labels: [ + labels: [{ op: '=', value: 'foo', label: 'project' }], + operations: [{ id: 'rate', params: ['5m'] }], + }, + }, + ], + }) + ); + }); + + it('parses binary scalar query', () => { + expect(buildVisualQueryFromString('rate({project="bar"}[5m]) / 2')).toEqual( + noErrors({ + labels: [{ op: '=', value: 'bar', label: 'project' }], + operations: [ + { id: 'rate', params: ['5m'] }, + { id: '__divide_by', params: [2] }, + ], + }) + ); + }); + + it('parses chained binary query', () => { + expect( + buildVisualQueryFromString('rate({project="bar"}[5m]) * 2 / rate({project="foo"}[5m]) + rate({app="test"}[1m])') + ).toEqual( + noErrors({ + labels: [{ op: '=', value: 'bar', label: 'project' }], + operations: [ + { id: 'rate', params: ['5m'] }, + { id: '__multiply_by', params: [2] }, + ], + binaryQueries: [ + { + operator: '/', + query: { + labels: [{ op: '=', value: 'foo', label: 'project' }], + operations: [{ id: 'rate', params: ['5m'] }], + binaryQueries: [ { - op: '=', - value: 'foo', - label: 'project', + operator: '+', + query: { + labels: [{ op: '=', value: 'test', label: 'app' }], + operations: [{ id: 'rate', params: ['1m'] }], + }, }, ], - operations: [{ id: 'rate', params: ['5m'] }], }, }, ], diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.ts index e1764833747..f6e58d73799 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.ts @@ -1,12 +1,17 @@ import { parser } from '@grafana/lezer-logql'; -import { SyntaxNode, TreeCursor } from '@lezer/common'; +import { SyntaxNode } from '@lezer/common'; +import { + ErrorName, + getLeftMostChild, + getString, + makeBinOp, + makeError, + replaceVariables, +} from '../../prometheus/querybuilder/shared/parsingUtils'; import { QueryBuilderLabelFilter, QueryBuilderOperation } from '../../prometheus/querybuilder/shared/types'; import { binaryScalarDefs } from './binaryScalarOperations'; import { LokiVisualQuery, LokiVisualQueryBinary } from './types'; -// This is used for error type -const ErrorName = '⚠'; - interface Context { query: LokiVisualQuery; errors: ParsingError[]; @@ -300,8 +305,8 @@ function handleBinary(expr: string, node: SyntaxNode, context: Context) { const opDef = operatorToOpName[op]; - const leftNumber = left.getChild('NumberLiteral'); - const rightNumber = right.getChild('NumberLiteral'); + const leftNumber = getLastChildWithSelector(left, 'MetricExpr.LiteralExpr.Number'); + const rightNumber = getLastChildWithSelector(right, 'MetricExpr.LiteralExpr.Number'); const rightBinary = right.getChild('BinOpExpr'); @@ -320,7 +325,7 @@ function handleBinary(expr: string, node: SyntaxNode, context: Context) { // Due to the way binary ops are parsed we can get a binary operation on the right that starts with a number which // is a factor for a current binary operation. So we have to add it as an operation now. const leftMostChild = getLeftMostChild(right); - if (leftMostChild?.name === 'NumberLiteral') { + if (leftMostChild?.name === 'Number') { visQuery.operations.push(makeBinOp(opDef, expr, leftMostChild, !!binModifier?.isBool)); } @@ -376,51 +381,6 @@ function getBinaryModifier( } } -function makeBinOp( - opDef: { id: string; comparison?: boolean }, - expr: string, - numberNode: SyntaxNode, - hasBool: boolean -) { - const params: any[] = [parseFloat(getString(expr, numberNode))]; - if (opDef.comparison) { - params.unshift(hasBool); - } - return { - id: opDef.id, - params, - }; -} - -function getLeftMostChild(cur: SyntaxNode): SyntaxNode | null { - let child = cur; - while (true) { - if (child.firstChild) { - child = child.firstChild; - } else { - break; - } - } - return child; -} - -function getString(expr: string, node: SyntaxNode | TreeCursor | null | undefined) { - if (!node) { - return ''; - } - - return returnVariables(expr.substring(node.from, node.to)); -} - -function makeError(expr: string, node: SyntaxNode) { - return { - text: getString(expr, node), - from: node.from, - to: node.to, - parentType: node.parent?.name, - }; -} - function isIntervalVariableError(node: SyntaxNode) { return node?.parent?.name === 'Range'; } @@ -432,53 +392,20 @@ function handleQuotes(string: string) { return string.replace(/`/g, ''); } -// Template variables -// Taken from template_srv, but copied so to not mess with the regex.index which is manipulated in the service -/* - * This regex matches 3 types of variable reference with an optional format specifier - * \$(\w+) $var1 - * \[\[([\s\S]+?)(?::(\w+))?\]\] [[var2]] or [[var2:fmt2]] - * \${(\w+)(?::(\w+))?} ${var3} or ${var3:fmt3} - */ -const variableRegex = /\$(\w+)|\[\[([\s\S]+?)(?::(\w+))?\]\]|\${(\w+)(?:\.([^:^\}]+))?(?::([^\}]+))?}/g; - /** - * As variables with $ are creating parsing errors, we first replace them with magic string that is parseable and at - * the same time we can get the variable and it's format back from it. - * @param expr + * Simple helper to traverse the syntax tree. Instead of node.getChild('foo')?.getChild('bar')?.getChild('baz') you + * can write getChildWithSelector(node, 'foo.bar.baz') + * @param node + * @param selector */ -function replaceVariables(expr: string) { - return expr.replace(variableRegex, (match, var1, var2, fmt2, var3, fieldPath, fmt3) => { - const fmt = fmt2 || fmt3; - let variable = var1; - let varType = '0'; - - if (var2) { - variable = var2; - varType = '1'; +function getLastChildWithSelector(node: SyntaxNode, selector: string) { + let child: SyntaxNode | null = node; + const children = selector.split('.'); + for (const s of children) { + child = child.getChild(s); + if (!child) { + return null; } - - if (var3) { - variable = var3; - varType = '2'; - } - - return `__V_${varType}__` + variable + '__V__' + (fmt ? '__F__' + fmt + '__F__' : ''); - }); -} - -const varTypeFunc = [ - (v: string, f?: string) => `\$${v}`, - (v: string, f?: string) => `[[${v}${f ? `:${f}` : ''}]]`, - (v: string, f?: string) => `\$\{${v}${f ? `:${f}` : ''}\}`, -]; - -/** - * Get beck the text with variables in their original format. - * @param expr - */ -function returnVariables(expr: string) { - return expr.replace(/__V_(\d)__(.+)__V__(?:__F__(\w+)__F__)?/g, (match, type, v, f) => { - return varTypeFunc[parseInt(type, 10)](v, f); - }); + } + return child; } diff --git a/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts b/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts index 8d1a7908bf3..c8a7d145ace 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts +++ b/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts @@ -1,58 +1,9 @@ import { parser } from 'lezer-promql'; -import { SyntaxNode, TreeCursor } from '@lezer/common'; +import { SyntaxNode } from '@lezer/common'; import { QueryBuilderLabelFilter, QueryBuilderOperation } from './shared/types'; import { PromVisualQuery, PromVisualQueryBinary } from './types'; import { binaryScalarDefs } from './binaryScalarOperations'; - -// Taken from template_srv, but copied so to not mess with the regex.index which is manipulated in the service -/* - * This regex matches 3 types of variable reference with an optional format specifier - * \$(\w+) $var1 - * \[\[([\s\S]+?)(?::(\w+))?\]\] [[var2]] or [[var2:fmt2]] - * \${(\w+)(?::(\w+))?} ${var3} or ${var3:fmt3} - */ -const variableRegex = /\$(\w+)|\[\[([\s\S]+?)(?::(\w+))?\]\]|\${(\w+)(?:\.([^:^\}]+))?(?::([^\}]+))?}/g; - -/** - * As variables with $ are creating parsing errors, we first replace them with magic string that is parseable and at - * the same time we can get the variable and it's format back from it. - * @param expr - */ -function replaceVariables(expr: string) { - return expr.replace(variableRegex, (match, var1, var2, fmt2, var3, fieldPath, fmt3) => { - const fmt = fmt2 || fmt3; - let variable = var1; - let varType = '0'; - - if (var2) { - variable = var2; - varType = '1'; - } - - if (var3) { - variable = var3; - varType = '2'; - } - - return `__V_${varType}__` + variable + '__V__' + (fmt ? '__F__' + fmt + '__F__' : ''); - }); -} - -const varTypeFunc = [ - (v: string, f?: string) => `\$${v}`, - (v: string, f?: string) => `[[${v}${f ? `:${f}` : ''}]]`, - (v: string, f?: string) => `\$\{${v}${f ? `:${f}` : ''}\}`, -]; - -/** - * Get beck the text with variables in their original format. - * @param expr - */ -function returnVariables(expr: string) { - return expr.replace(/__V_(\d)__(.+)__V__(?:__F__(\w+)__F__)?/g, (match, type, v, f) => { - return varTypeFunc[parseInt(type, 10)](v, f); - }); -} +import { ErrorName, getLeftMostChild, getString, makeBinOp, makeError, replaceVariables } from './shared/parsingUtils'; /** * Parses a PromQL query into a visual query model. @@ -102,9 +53,6 @@ interface Context { errors: ParsingError[]; } -// This is used for error type for some reason -const ErrorName = '⚠'; - /** * Handler for default state. It will traverse the tree and call the appropriate handler for each node. The node * handled here does not necessarily need to be of type == Expr. @@ -173,18 +121,6 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex } } -function makeError(expr: string, node: SyntaxNode) { - return { - text: getString(expr, node), - // TODO: this are positions in the string with the replaced variables. Means it cannot be used to show exact - // placement of the error for the user. We need some translation table to positions before the variable - // replace. - from: node.from, - to: node.to, - parentType: node.parent?.name, - }; -} - function isIntervalVariableError(node: SyntaxNode) { return node.prevSibling?.name === 'Expr' && node.prevSibling?.firstChild?.name === 'VectorSelector'; } @@ -430,35 +366,6 @@ function getBinaryModifier( } } -function makeBinOp( - opDef: { id: string; comparison?: boolean }, - expr: string, - numberNode: SyntaxNode, - hasBool: boolean -) { - const params: any[] = [parseFloat(getString(expr, numberNode))]; - if (opDef.comparison) { - params.push(hasBool); - } - return { - id: opDef.id, - params, - }; -} - -/** - * Get the actual string of the expression. That is not stored in the tree so we have to get the indexes from the node - * and then based on that get it from the expression. - * @param expr - * @param node - */ -function getString(expr: string, node: SyntaxNode | TreeCursor | null | undefined) { - if (!node) { - return ''; - } - return returnVariables(expr.substring(node.from, node.to)); -} - /** * Get all nodes with type in the tree. This traverses the tree so it is safe only when you know there shouldn't be * too much nesting but you just want to skip some of the wrappers. For example getting function args this way would @@ -481,50 +388,3 @@ function getAllByType(expr: string, cur: SyntaxNode, type: string): string[] { } return values; } - -function getLeftMostChild(cur: SyntaxNode): SyntaxNode | null { - let child = cur; - while (true) { - if (child.firstChild) { - child = child.firstChild; - } else { - break; - } - } - return child; -} - -// Debugging function for convenience. -// @ts-ignore -function log(expr: string, cur?: SyntaxNode) { - const json = toJson(expr, cur); - if (!json) { - console.log(''); - return; - } - console.log(JSON.stringify(json, undefined, 2)); -} - -function toJson(expr: string, cur?: SyntaxNode) { - if (!cur) { - return undefined; - } - const treeJson: any = {}; - const name = nodeToString(expr, cur); - const children = []; - - let pos = 0; - let child = cur.childAfter(pos); - while (child) { - children.push(toJson(expr, child)); - pos = child.to; - child = cur.childAfter(pos); - } - - treeJson[name] = children; - return treeJson; -} - -function nodeToString(expr: string, node: SyntaxNode) { - return node.name + ':' + getString(expr, node); -} diff --git a/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.test.ts b/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.test.ts new file mode 100644 index 00000000000..d5b8804a167 --- /dev/null +++ b/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.test.ts @@ -0,0 +1,41 @@ +import { getLeftMostChild, getString, replaceVariables } from './parsingUtils'; +import { parser } from 'lezer-promql'; + +describe('getLeftMostChild', () => { + it('return left most child', () => { + const tree = parser.parse('sum_over_time(foo{bar="baz"}[5m])'); + const child = getLeftMostChild(tree.topNode); + expect(child).toBeDefined(); + expect(child!.name).toBe('SumOverTime'); + }); +}); + +describe('replaceVariables', () => { + it('should replace variables', () => { + expect(replaceVariables('sum_over_time([[metric_var]]{bar="${app}"}[$__interval])')).toBe( + 'sum_over_time(__V_1__metric_var__V__{bar="__V_2__app__V__"}[__V_0____interval__V__])' + ); + }); +}); + +describe('getString', () => { + it('should return correct string representation of the node', () => { + const expr = 'sum_over_time(foo{bar="baz"}[5m])'; + const tree = parser.parse(expr); + const child = getLeftMostChild(tree.topNode); + expect(getString(expr, child)).toBe('sum_over_time'); + }); + + it('should return string with correct variables', () => { + const expr = 'sum_over_time(__V_1__metric_var__V__{bar="__V_2__app__V__"}[__V_0____interval__V__])'; + const tree = parser.parse(expr); + expect(getString(expr, tree.topNode)).toBe('sum_over_time([[metric_var]]{bar="${app}"}[$__interval])'); + }); + + it('is symmetrical with replaceVariables', () => { + const expr = 'sum_over_time([[metric_var]]{bar="${app}"}[$__interval])'; + const replaced = replaceVariables(expr); + const tree = parser.parse(replaced); + expect(getString(replaced, tree.topNode)).toBe(expr); + }); +}); diff --git a/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts b/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts new file mode 100644 index 00000000000..8fa3db87621 --- /dev/null +++ b/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts @@ -0,0 +1,177 @@ +import { SyntaxNode, TreeCursor } from '@lezer/common'; +import { QueryBuilderOperation } from './types'; + +// This is used for error type for some reason +export const ErrorName = '⚠'; + +export function getLeftMostChild(cur: SyntaxNode): SyntaxNode { + return cur.firstChild ? getLeftMostChild(cur.firstChild) : cur; +} + +export function makeError(expr: string, node: SyntaxNode) { + return { + text: getString(expr, node), + // TODO: this are positions in the string with the replaced variables. Means it cannot be used to show exact + // placement of the error for the user. We need some translation table to positions before the variable + // replace. + from: node.from, + to: node.to, + parentType: node.parent?.name, + }; +} + +// Taken from template_srv, but copied so to not mess with the regex.index which is manipulated in the service +/* + * This regex matches 3 types of variable reference with an optional format specifier + * \$(\w+) $var1 + * \[\[([\s\S]+?)(?::(\w+))?\]\] [[var2]] or [[var2:fmt2]] + * \${(\w+)(?::(\w+))?} ${var3} or ${var3:fmt3} + */ +const variableRegex = /\$(\w+)|\[\[([\s\S]+?)(?::(\w+))?\]\]|\${(\w+)(?:\.([^:^\}]+))?(?::([^\}]+))?}/g; + +/** + * As variables with $ are creating parsing errors, we first replace them with magic string that is parseable and at + * the same time we can get the variable and it's format back from it. + * @param expr + */ +export function replaceVariables(expr: string) { + return expr.replace(variableRegex, (match, var1, var2, fmt2, var3, fieldPath, fmt3) => { + const fmt = fmt2 || fmt3; + let variable = var1; + let varType = '0'; + + if (var2) { + variable = var2; + varType = '1'; + } + + if (var3) { + variable = var3; + varType = '2'; + } + + return `__V_${varType}__` + variable + '__V__' + (fmt ? '__F__' + fmt + '__F__' : ''); + }); +} + +const varTypeFunc = [ + (v: string, f?: string) => `\$${v}`, + (v: string, f?: string) => `[[${v}${f ? `:${f}` : ''}]]`, + (v: string, f?: string) => `\$\{${v}${f ? `:${f}` : ''}\}`, +]; + +/** + * Get back the text with variables in their original format. + * @param expr + */ +function returnVariables(expr: string) { + return expr.replace(/__V_(\d)__(.+?)__V__(?:__F__(\w+)__F__)?/g, (match, type, v, f) => { + return varTypeFunc[parseInt(type, 10)](v, f); + }); +} + +/** + * Get the actual string of the expression. That is not stored in the tree so we have to get the indexes from the node + * and then based on that get it from the expression. + * @param expr + * @param node + */ +export function getString(expr: string, node: SyntaxNode | TreeCursor | null | undefined) { + if (!node) { + return ''; + } + return returnVariables(expr.substring(node.from, node.to)); +} + +/** + * Create simple scalar binary op object. + * @param opDef - definition of the op to be created + * @param expr + * @param numberNode - the node for the scalar + * @param hasBool - whether operation has a bool modifier. Is used only for ops for which it makes sense. + */ +export function makeBinOp( + opDef: { id: string; comparison?: boolean }, + expr: string, + numberNode: SyntaxNode, + hasBool: boolean +): QueryBuilderOperation { + const params: any[] = [parseFloat(getString(expr, numberNode))]; + if (opDef.comparison) { + params.push(hasBool); + } + return { + id: opDef.id, + params, + }; +} + +// Debugging function for convenience. Gives you nice output similar to linux tree util. +// @ts-ignore +export function log(expr: string, cur?: SyntaxNode) { + if (!cur) { + console.log(''); + return; + } + const json = toJson(expr, cur); + const text = jsonToText(json); + + if (!text) { + console.log(''); + return; + } + console.log(text); +} + +function toJson(expr: string, cur: SyntaxNode) { + const treeJson: any = {}; + const name = nodeToString(expr, cur); + const children = []; + + let pos = 0; + let child = cur.childAfter(pos); + while (child) { + children.push(toJson(expr, child)); + pos = child.to; + child = cur.childAfter(pos); + } + + treeJson.name = name; + treeJson.children = children; + return treeJson; +} + +type JsonNode = { + name: string; + children: JsonNode[]; +}; + +function jsonToText( + node: JsonNode, + context: { lastChild: boolean; indent: string } = { + lastChild: true, + indent: '', + } +) { + const name = node.name; + const { lastChild, indent } = context; + const newIndent = indent !== '' ? indent + (lastChild ? '└─' : '├─') : ''; + let text = newIndent + name; + + const children = node.children; + children.forEach((child: any, index: number) => { + const isLastChild = index === children.length - 1; + text += + '\n' + + jsonToText(child, { + lastChild: isLastChild, + indent: indent + (lastChild ? ' ' : '│ '), + }); + }); + + return text; +} + +function nodeToString(expr: string, node: SyntaxNode) { + return node.name + ': ' + getString(expr, node); +}