Loki: Fix parsing chained and scalar binary queries (#47132)

* Fix bin parsing for more cases

* Add test for some utils

* Removing console.log

* Simplify some code and fix typing

* Rename function
This commit is contained in:
Andrej Ocenas 2022-04-01 18:33:09 +02:00 committed by GitHub
parent ba2332eb96
commit 1110666b6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 289 additions and 252 deletions

View File

@ -290,26 +290,58 @@ describe('buildVisualQueryFromString', () => {
it('parses binary query', () => {
expect(buildVisualQueryFromString('rate({project="bar"}[5m]) / rate({project="foo"}[5m])')).toEqual(
noErrors({
labels: [
{
op: '=',
value: 'bar',
label: 'project',
},
],
labels: [{ op: '=', value: 'bar', label: 'project' }],
operations: [{ id: 'rate', params: ['5m'] }],
binaryQueries: [
{
operator: '/',
query: {
labels: [
labels: [{ op: '=', value: 'foo', label: 'project' }],
operations: [{ id: 'rate', params: ['5m'] }],
},
},
],
})
);
});
it('parses binary scalar query', () => {
expect(buildVisualQueryFromString('rate({project="bar"}[5m]) / 2')).toEqual(
noErrors({
labels: [{ op: '=', value: 'bar', label: 'project' }],
operations: [
{ id: 'rate', params: ['5m'] },
{ id: '__divide_by', params: [2] },
],
})
);
});
it('parses chained binary query', () => {
expect(
buildVisualQueryFromString('rate({project="bar"}[5m]) * 2 / rate({project="foo"}[5m]) + rate({app="test"}[1m])')
).toEqual(
noErrors({
labels: [{ op: '=', value: 'bar', label: 'project' }],
operations: [
{ id: 'rate', params: ['5m'] },
{ id: '__multiply_by', params: [2] },
],
binaryQueries: [
{
operator: '/',
query: {
labels: [{ op: '=', value: 'foo', label: 'project' }],
operations: [{ id: 'rate', params: ['5m'] }],
binaryQueries: [
{
op: '=',
value: 'foo',
label: 'project',
operator: '+',
query: {
labels: [{ op: '=', value: 'test', label: 'app' }],
operations: [{ id: 'rate', params: ['1m'] }],
},
},
],
operations: [{ id: 'rate', params: ['5m'] }],
},
},
],

View File

@ -1,12 +1,17 @@
import { parser } from '@grafana/lezer-logql';
import { SyntaxNode, TreeCursor } from '@lezer/common';
import { SyntaxNode } from '@lezer/common';
import {
ErrorName,
getLeftMostChild,
getString,
makeBinOp,
makeError,
replaceVariables,
} from '../../prometheus/querybuilder/shared/parsingUtils';
import { QueryBuilderLabelFilter, QueryBuilderOperation } from '../../prometheus/querybuilder/shared/types';
import { binaryScalarDefs } from './binaryScalarOperations';
import { LokiVisualQuery, LokiVisualQueryBinary } from './types';
// This is used for error type
const ErrorName = '⚠';
interface Context {
query: LokiVisualQuery;
errors: ParsingError[];
@ -300,8 +305,8 @@ function handleBinary(expr: string, node: SyntaxNode, context: Context) {
const opDef = operatorToOpName[op];
const leftNumber = left.getChild('NumberLiteral');
const rightNumber = right.getChild('NumberLiteral');
const leftNumber = getLastChildWithSelector(left, 'MetricExpr.LiteralExpr.Number');
const rightNumber = getLastChildWithSelector(right, 'MetricExpr.LiteralExpr.Number');
const rightBinary = right.getChild('BinOpExpr');
@ -320,7 +325,7 @@ function handleBinary(expr: string, node: SyntaxNode, context: Context) {
// Due to the way binary ops are parsed we can get a binary operation on the right that starts with a number which
// is a factor for a current binary operation. So we have to add it as an operation now.
const leftMostChild = getLeftMostChild(right);
if (leftMostChild?.name === 'NumberLiteral') {
if (leftMostChild?.name === 'Number') {
visQuery.operations.push(makeBinOp(opDef, expr, leftMostChild, !!binModifier?.isBool));
}
@ -376,51 +381,6 @@ function getBinaryModifier(
}
}
function makeBinOp(
opDef: { id: string; comparison?: boolean },
expr: string,
numberNode: SyntaxNode,
hasBool: boolean
) {
const params: any[] = [parseFloat(getString(expr, numberNode))];
if (opDef.comparison) {
params.unshift(hasBool);
}
return {
id: opDef.id,
params,
};
}
function getLeftMostChild(cur: SyntaxNode): SyntaxNode | null {
let child = cur;
while (true) {
if (child.firstChild) {
child = child.firstChild;
} else {
break;
}
}
return child;
}
function getString(expr: string, node: SyntaxNode | TreeCursor | null | undefined) {
if (!node) {
return '';
}
return returnVariables(expr.substring(node.from, node.to));
}
function makeError(expr: string, node: SyntaxNode) {
return {
text: getString(expr, node),
from: node.from,
to: node.to,
parentType: node.parent?.name,
};
}
function isIntervalVariableError(node: SyntaxNode) {
return node?.parent?.name === 'Range';
}
@ -432,53 +392,20 @@ function handleQuotes(string: string) {
return string.replace(/`/g, '');
}
// Template variables
// Taken from template_srv, but copied so to not mess with the regex.index which is manipulated in the service
/*
* This regex matches 3 types of variable reference with an optional format specifier
* \$(\w+) $var1
* \[\[([\s\S]+?)(?::(\w+))?\]\] [[var2]] or [[var2:fmt2]]
* \${(\w+)(?::(\w+))?} ${var3} or ${var3:fmt3}
*/
const variableRegex = /\$(\w+)|\[\[([\s\S]+?)(?::(\w+))?\]\]|\${(\w+)(?:\.([^:^\}]+))?(?::([^\}]+))?}/g;
/**
* As variables with $ are creating parsing errors, we first replace them with magic string that is parseable and at
* the same time we can get the variable and it's format back from it.
* @param expr
* Simple helper to traverse the syntax tree. Instead of node.getChild('foo')?.getChild('bar')?.getChild('baz') you
* can write getChildWithSelector(node, 'foo.bar.baz')
* @param node
* @param selector
*/
function replaceVariables(expr: string) {
return expr.replace(variableRegex, (match, var1, var2, fmt2, var3, fieldPath, fmt3) => {
const fmt = fmt2 || fmt3;
let variable = var1;
let varType = '0';
if (var2) {
variable = var2;
varType = '1';
function getLastChildWithSelector(node: SyntaxNode, selector: string) {
let child: SyntaxNode | null = node;
const children = selector.split('.');
for (const s of children) {
child = child.getChild(s);
if (!child) {
return null;
}
if (var3) {
variable = var3;
varType = '2';
}
return `__V_${varType}__` + variable + '__V__' + (fmt ? '__F__' + fmt + '__F__' : '');
});
}
const varTypeFunc = [
(v: string, f?: string) => `\$${v}`,
(v: string, f?: string) => `[[${v}${f ? `:${f}` : ''}]]`,
(v: string, f?: string) => `\$\{${v}${f ? `:${f}` : ''}\}`,
];
/**
* Get beck the text with variables in their original format.
* @param expr
*/
function returnVariables(expr: string) {
return expr.replace(/__V_(\d)__(.+)__V__(?:__F__(\w+)__F__)?/g, (match, type, v, f) => {
return varTypeFunc[parseInt(type, 10)](v, f);
});
}
return child;
}

View File

@ -1,58 +1,9 @@
import { parser } from 'lezer-promql';
import { SyntaxNode, TreeCursor } from '@lezer/common';
import { SyntaxNode } from '@lezer/common';
import { QueryBuilderLabelFilter, QueryBuilderOperation } from './shared/types';
import { PromVisualQuery, PromVisualQueryBinary } from './types';
import { binaryScalarDefs } from './binaryScalarOperations';
// Taken from template_srv, but copied so to not mess with the regex.index which is manipulated in the service
/*
* This regex matches 3 types of variable reference with an optional format specifier
* \$(\w+) $var1
* \[\[([\s\S]+?)(?::(\w+))?\]\] [[var2]] or [[var2:fmt2]]
* \${(\w+)(?::(\w+))?} ${var3} or ${var3:fmt3}
*/
const variableRegex = /\$(\w+)|\[\[([\s\S]+?)(?::(\w+))?\]\]|\${(\w+)(?:\.([^:^\}]+))?(?::([^\}]+))?}/g;
/**
* As variables with $ are creating parsing errors, we first replace them with magic string that is parseable and at
* the same time we can get the variable and it's format back from it.
* @param expr
*/
function replaceVariables(expr: string) {
return expr.replace(variableRegex, (match, var1, var2, fmt2, var3, fieldPath, fmt3) => {
const fmt = fmt2 || fmt3;
let variable = var1;
let varType = '0';
if (var2) {
variable = var2;
varType = '1';
}
if (var3) {
variable = var3;
varType = '2';
}
return `__V_${varType}__` + variable + '__V__' + (fmt ? '__F__' + fmt + '__F__' : '');
});
}
const varTypeFunc = [
(v: string, f?: string) => `\$${v}`,
(v: string, f?: string) => `[[${v}${f ? `:${f}` : ''}]]`,
(v: string, f?: string) => `\$\{${v}${f ? `:${f}` : ''}\}`,
];
/**
* Get beck the text with variables in their original format.
* @param expr
*/
function returnVariables(expr: string) {
return expr.replace(/__V_(\d)__(.+)__V__(?:__F__(\w+)__F__)?/g, (match, type, v, f) => {
return varTypeFunc[parseInt(type, 10)](v, f);
});
}
import { ErrorName, getLeftMostChild, getString, makeBinOp, makeError, replaceVariables } from './shared/parsingUtils';
/**
* Parses a PromQL query into a visual query model.
@ -102,9 +53,6 @@ interface Context {
errors: ParsingError[];
}
// This is used for error type for some reason
const ErrorName = '⚠';
/**
* Handler for default state. It will traverse the tree and call the appropriate handler for each node. The node
* handled here does not necessarily need to be of type == Expr.
@ -173,18 +121,6 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex
}
}
function makeError(expr: string, node: SyntaxNode) {
return {
text: getString(expr, node),
// TODO: this are positions in the string with the replaced variables. Means it cannot be used to show exact
// placement of the error for the user. We need some translation table to positions before the variable
// replace.
from: node.from,
to: node.to,
parentType: node.parent?.name,
};
}
function isIntervalVariableError(node: SyntaxNode) {
return node.prevSibling?.name === 'Expr' && node.prevSibling?.firstChild?.name === 'VectorSelector';
}
@ -430,35 +366,6 @@ function getBinaryModifier(
}
}
function makeBinOp(
opDef: { id: string; comparison?: boolean },
expr: string,
numberNode: SyntaxNode,
hasBool: boolean
) {
const params: any[] = [parseFloat(getString(expr, numberNode))];
if (opDef.comparison) {
params.push(hasBool);
}
return {
id: opDef.id,
params,
};
}
/**
* Get the actual string of the expression. That is not stored in the tree so we have to get the indexes from the node
* and then based on that get it from the expression.
* @param expr
* @param node
*/
function getString(expr: string, node: SyntaxNode | TreeCursor | null | undefined) {
if (!node) {
return '';
}
return returnVariables(expr.substring(node.from, node.to));
}
/**
* Get all nodes with type in the tree. This traverses the tree so it is safe only when you know there shouldn't be
* too much nesting but you just want to skip some of the wrappers. For example getting function args this way would
@ -481,50 +388,3 @@ function getAllByType(expr: string, cur: SyntaxNode, type: string): string[] {
}
return values;
}
function getLeftMostChild(cur: SyntaxNode): SyntaxNode | null {
let child = cur;
while (true) {
if (child.firstChild) {
child = child.firstChild;
} else {
break;
}
}
return child;
}
// Debugging function for convenience.
// @ts-ignore
function log(expr: string, cur?: SyntaxNode) {
const json = toJson(expr, cur);
if (!json) {
console.log('<empty>');
return;
}
console.log(JSON.stringify(json, undefined, 2));
}
function toJson(expr: string, cur?: SyntaxNode) {
if (!cur) {
return undefined;
}
const treeJson: any = {};
const name = nodeToString(expr, cur);
const children = [];
let pos = 0;
let child = cur.childAfter(pos);
while (child) {
children.push(toJson(expr, child));
pos = child.to;
child = cur.childAfter(pos);
}
treeJson[name] = children;
return treeJson;
}
function nodeToString(expr: string, node: SyntaxNode) {
return node.name + ':' + getString(expr, node);
}

View File

@ -0,0 +1,41 @@
import { getLeftMostChild, getString, replaceVariables } from './parsingUtils';
import { parser } from 'lezer-promql';
describe('getLeftMostChild', () => {
it('return left most child', () => {
const tree = parser.parse('sum_over_time(foo{bar="baz"}[5m])');
const child = getLeftMostChild(tree.topNode);
expect(child).toBeDefined();
expect(child!.name).toBe('SumOverTime');
});
});
describe('replaceVariables', () => {
it('should replace variables', () => {
expect(replaceVariables('sum_over_time([[metric_var]]{bar="${app}"}[$__interval])')).toBe(
'sum_over_time(__V_1__metric_var__V__{bar="__V_2__app__V__"}[__V_0____interval__V__])'
);
});
});
describe('getString', () => {
it('should return correct string representation of the node', () => {
const expr = 'sum_over_time(foo{bar="baz"}[5m])';
const tree = parser.parse(expr);
const child = getLeftMostChild(tree.topNode);
expect(getString(expr, child)).toBe('sum_over_time');
});
it('should return string with correct variables', () => {
const expr = 'sum_over_time(__V_1__metric_var__V__{bar="__V_2__app__V__"}[__V_0____interval__V__])';
const tree = parser.parse(expr);
expect(getString(expr, tree.topNode)).toBe('sum_over_time([[metric_var]]{bar="${app}"}[$__interval])');
});
it('is symmetrical with replaceVariables', () => {
const expr = 'sum_over_time([[metric_var]]{bar="${app}"}[$__interval])';
const replaced = replaceVariables(expr);
const tree = parser.parse(replaced);
expect(getString(replaced, tree.topNode)).toBe(expr);
});
});

View File

@ -0,0 +1,177 @@
import { SyntaxNode, TreeCursor } from '@lezer/common';
import { QueryBuilderOperation } from './types';
// This is used for error type for some reason
export const ErrorName = '⚠';
export function getLeftMostChild(cur: SyntaxNode): SyntaxNode {
return cur.firstChild ? getLeftMostChild(cur.firstChild) : cur;
}
export function makeError(expr: string, node: SyntaxNode) {
return {
text: getString(expr, node),
// TODO: this are positions in the string with the replaced variables. Means it cannot be used to show exact
// placement of the error for the user. We need some translation table to positions before the variable
// replace.
from: node.from,
to: node.to,
parentType: node.parent?.name,
};
}
// Taken from template_srv, but copied so to not mess with the regex.index which is manipulated in the service
/*
* This regex matches 3 types of variable reference with an optional format specifier
* \$(\w+) $var1
* \[\[([\s\S]+?)(?::(\w+))?\]\] [[var2]] or [[var2:fmt2]]
* \${(\w+)(?::(\w+))?} ${var3} or ${var3:fmt3}
*/
const variableRegex = /\$(\w+)|\[\[([\s\S]+?)(?::(\w+))?\]\]|\${(\w+)(?:\.([^:^\}]+))?(?::([^\}]+))?}/g;
/**
* As variables with $ are creating parsing errors, we first replace them with magic string that is parseable and at
* the same time we can get the variable and it's format back from it.
* @param expr
*/
export function replaceVariables(expr: string) {
return expr.replace(variableRegex, (match, var1, var2, fmt2, var3, fieldPath, fmt3) => {
const fmt = fmt2 || fmt3;
let variable = var1;
let varType = '0';
if (var2) {
variable = var2;
varType = '1';
}
if (var3) {
variable = var3;
varType = '2';
}
return `__V_${varType}__` + variable + '__V__' + (fmt ? '__F__' + fmt + '__F__' : '');
});
}
const varTypeFunc = [
(v: string, f?: string) => `\$${v}`,
(v: string, f?: string) => `[[${v}${f ? `:${f}` : ''}]]`,
(v: string, f?: string) => `\$\{${v}${f ? `:${f}` : ''}\}`,
];
/**
* Get back the text with variables in their original format.
* @param expr
*/
function returnVariables(expr: string) {
return expr.replace(/__V_(\d)__(.+?)__V__(?:__F__(\w+)__F__)?/g, (match, type, v, f) => {
return varTypeFunc[parseInt(type, 10)](v, f);
});
}
/**
* Get the actual string of the expression. That is not stored in the tree so we have to get the indexes from the node
* and then based on that get it from the expression.
* @param expr
* @param node
*/
export function getString(expr: string, node: SyntaxNode | TreeCursor | null | undefined) {
if (!node) {
return '';
}
return returnVariables(expr.substring(node.from, node.to));
}
/**
* Create simple scalar binary op object.
* @param opDef - definition of the op to be created
* @param expr
* @param numberNode - the node for the scalar
* @param hasBool - whether operation has a bool modifier. Is used only for ops for which it makes sense.
*/
export function makeBinOp(
opDef: { id: string; comparison?: boolean },
expr: string,
numberNode: SyntaxNode,
hasBool: boolean
): QueryBuilderOperation {
const params: any[] = [parseFloat(getString(expr, numberNode))];
if (opDef.comparison) {
params.push(hasBool);
}
return {
id: opDef.id,
params,
};
}
// Debugging function for convenience. Gives you nice output similar to linux tree util.
// @ts-ignore
export function log(expr: string, cur?: SyntaxNode) {
if (!cur) {
console.log('<empty>');
return;
}
const json = toJson(expr, cur);
const text = jsonToText(json);
if (!text) {
console.log('<empty>');
return;
}
console.log(text);
}
function toJson(expr: string, cur: SyntaxNode) {
const treeJson: any = {};
const name = nodeToString(expr, cur);
const children = [];
let pos = 0;
let child = cur.childAfter(pos);
while (child) {
children.push(toJson(expr, child));
pos = child.to;
child = cur.childAfter(pos);
}
treeJson.name = name;
treeJson.children = children;
return treeJson;
}
type JsonNode = {
name: string;
children: JsonNode[];
};
function jsonToText(
node: JsonNode,
context: { lastChild: boolean; indent: string } = {
lastChild: true,
indent: '',
}
) {
const name = node.name;
const { lastChild, indent } = context;
const newIndent = indent !== '' ? indent + (lastChild ? '└─' : '├─') : '';
let text = newIndent + name;
const children = node.children;
children.forEach((child: any, index: number) => {
const isLastChild = index === children.length - 1;
text +=
'\n' +
jsonToText(child, {
lastChild: isLastChild,
indent: indent + (lastChild ? ' ' : '│ '),
});
});
return text;
}
function nodeToString(expr: string, node: SyntaxNode) {
return node.name + ': ' + getString(expr, node);
}