IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Prometheus: Update lezer-promql package (#85942)

Browse Source 
* Update @lezer/lr to v1.4.0

* Update @prometheus-io/lezer-promql to v0.37.0

* Update @prometheus-io/lezer-promql to v0.38.0

* Update @prometheus-io/lezer-promql to v0.39.0

* Update @prometheus-io/lezer-promql to v0.40.0

* add jest config

* update code

* fix code to pass "handles things" test

* fix retrieving labels

* fix code to pass "handles label values" test

* fix code to pass "simple binary comparison" test

* use BoolModifier

* add changed lines as comments

* fix for ambiguous query parsing tests

* resolve rebase conflict

* fix retrieving labels, aggregation with/out labels

* add error

* fix comment

* fix "reports error on parenthesis" unit test

* fix for "handles binary operation with vector matchers" test

* fix for "handles multiple binary scalar operations" test

* fix for "parses query without metric" test

* fix indentation and import style

* remove commented lines

* add todo items and comments

* remove dependency update from tempo datasource

* apply same changes in core prometheus frontend

* prettier

* add new test case

* use old version of lezer in the root package.json

* Revert "apply same changes in core prometheus frontend"

This reverts commit 83fd6ac7

* fix indentation

* use latest version of lezer-promql v0.51.2

* Update packages/grafana-prometheus/src/querybuilder/parsing.ts

Co-authored-by: Nick Richmond <5732000+NWRichmond@users.noreply.github.com>

* enable native histogram test

---------

Co-authored-by: Nick Richmond <5732000+NWRichmond@users.noreply.github.com>
This commit is contained in:
ismail simsek 2024-04-19 11:54:56 +02:00 committed by GitHub
parent 73873f5a8a
commit f9a8e34b32
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 114 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/grafana-prometheus/package.json
View File

@ -47,8 +47,8 @@
"@leeoniya/ufuzzy": "1.0.14",
"@lezer/common": "1.2.1",
"@lezer/highlight": "1.2.0",
"@lezer/lr": "1.3.3",
"@prometheus-io/lezer-promql": "^0.37.0-rc.1",
"@lezer/lr": "1.4.0",
"@prometheus-io/lezer-promql": "0.51.2",
"@reduxjs/toolkit": "1.9.5",
"d3": "7.9.0",
"date-fns": "3.6.0",

13
packages/grafana-prometheus/src/components/monaco-query-field/monaco-completion-provider/situation.test.ts
View File

@ -183,4 +183,17 @@ describe('situation', () => {
],
});
});
it('identifies all labels from queries when cursor is in middle', () => {
// Note the extra whitespace, if the cursor is after whitespace, the situation will fail to resolve
assertSituation('{one="val1", ^,two!="val2",three=~"val3",four!~"val4"}', {
type: 'IN_LABEL_SELECTOR_NO_LABEL_NAME',
otherLabels: [
{ name: 'one', value: 'val1', op: '=' },
{ name: 'two', value: 'val2', op: '!=' },
{ name: 'three', value: 'val3', op: '=~' },
{ name: 'four', value: 'val4', op: '!~' },
],
});
});
});

106
packages/grafana-prometheus/src/components/monaco-query-field/monaco-completion-provider/situation.ts
View File

@ -3,6 +3,7 @@ import type { SyntaxNode, Tree } from '@lezer/common';
import {
AggregateExpr,
AggregateModifier,
BinaryExpr,
EqlRegex,
EqlSingle,
FunctionCallBody,
@ -10,11 +11,9 @@ import {
Identifier,
LabelMatcher,
LabelMatchers,
LabelMatchList,
LabelName,
MatchOp,
MatrixSelector,
MetricIdentifier,
Neq,
NeqRegex,
parser,
@ -36,9 +35,7 @@ type NodeTypeId =
| typeof Identifier
| typeof LabelMatcher
| typeof LabelMatchers
| typeof LabelMatchList
| typeof LabelName
| typeof MetricIdentifier
| typeof PromQL
| typeof StringLiteral
| typeof VectorSelector
@ -184,6 +181,10 @@ const RESOLVERS: Resolver[] = [
path: [StringLiteral, LabelMatcher],
fun: resolveLabelMatcher,
},
{
path: [ERROR_NODE_NAME, BinaryExpr, PromQL],
fun: resolveTopLevel,
},
{
path: [ERROR_NODE_NAME, LabelMatcher],
fun: resolveLabelMatcher,
@ -252,30 +253,8 @@ function getLabels(labelMatchersNode: SyntaxNode, text: string): Label[] {
return [];
}
let listNode: SyntaxNode | null = walk(labelMatchersNode, [['firstChild', LabelMatchList]]);
const labels: Label[] = [];
while (listNode !== null) {
const matcherNode = walk(listNode, [['lastChild', LabelMatcher]]);
if (matcherNode === null) {
// unexpected, we stop
return [];
}
const label = getLabel(matcherNode, text);
if (label !== null) {
labels.push(label);
}
// there might be more labels
listNode = walk(listNode, [['firstChild', LabelMatchList]]);
}
// our labels-list is last-first, so we reverse it
labels.reverse();
return labels;
const labelNodes = labelMatchersNode.getChildren(LabelMatcher);
return labelNodes.map((ln) => getLabel(ln, text)).filter(notEmpty);
}
function getNodeChildren(node: SyntaxNode): SyntaxNode[] {
@ -319,17 +298,12 @@ function resolveLabelsForGrouping(node: SyntaxNode, text: string, pos: number):
return null;
}
const metricIdNode = getNodeInSubtree(bodyNode, MetricIdentifier);
const metricIdNode = getNodeInSubtree(bodyNode, Identifier);
if (metricIdNode === null) {
return null;
}
const idNode = walk(metricIdNode, [['firstChild', Identifier]]);
if (idNode === null) {
return null;
}
const metricName = getNodeText(idNode, text);
const metricName = getNodeText(metricIdNode, text);
return {
type: 'IN_GROUPING',
metricName,
@ -355,44 +329,11 @@ function resolveLabelMatcher(node: SyntaxNode, text: string, pos: number): Situa
const labelName = getNodeText(labelNameNode, text);
// now we need to go up, to the parent of LabelMatcher,
// there can be one or many `LabelMatchList` parents, we have
// to go through all of them
const firstListNode = walk(parent, [['parent', LabelMatchList]]);
if (firstListNode === null) {
const labelMatchersNode = walk(parent, [['parent', LabelMatchers]]);
if (labelMatchersNode === null) {
return null;
}
let listNode = firstListNode;
// we keep going through the parent-nodes
// as long as they are LabelMatchList.
// as soon as we reawch LabelMatchers, we stop
let labelMatchersNode: SyntaxNode | null = null;
while (labelMatchersNode === null) {
const p = listNode.parent;
if (p === null) {
return null;
}
const { id } = p.type;
switch (id) {
case LabelMatchList:
//we keep looping
listNode = p;
continue;
case LabelMatchers:
// we reached the end, we can stop the loop
labelMatchersNode = p;
continue;
default:
// we reached some other node, we stop
return null;
}
}
// now we need to find the other names
const allLabels = getLabels(labelMatchersNode, text);
@ -401,7 +342,6 @@ function resolveLabelMatcher(node: SyntaxNode, text: string, pos: number): Situa
const metricNameNode = walk(labelMatchersNode, [
['parent', VectorSelector],
['firstChild', MetricIdentifier],
['firstChild', Identifier],
]);
@ -444,23 +384,10 @@ function resolveDurations(node: SyntaxNode, text: string, pos: number): Situatio
};
}
function subTreeHasError(node: SyntaxNode): boolean {
return getNodeInSubtree(node, ERROR_NODE_NAME) !== null;
}
function resolveLabelKeysWithEquals(node: SyntaxNode, text: string, pos: number): Situation | null {
// for example `something{^}`
// there are some false positives that can end up in this situation, that we want
// to eliminate:
// `something{a~^}` (if this subtree contains any error-node, we stop)
if (subTreeHasError(node)) {
return null;
}
// next false positive:
// `something{a="1"^}`
const child = walk(node, [['firstChild', LabelMatchList]]);
const child = walk(node, [['firstChild', LabelMatcher]]);
if (child !== null) {
// means the label-matching part contains at least one label already.
//
@ -477,7 +404,6 @@ function resolveLabelKeysWithEquals(node: SyntaxNode, text: string, pos: number)
const metricNameNode = walk(node, [
['parent', VectorSelector],
['firstChild', MetricIdentifier],
['firstChild', Identifier],
]);
@ -533,7 +459,7 @@ export function getSituation(text: string, pos: number): Situation | null {
};
}
/*
/**
PromQL
Expr
VectorSelector
@ -546,7 +472,6 @@ export function getSituation(text: string, pos: number): Situation | null {
// also, if there are errors, the node lezer finds us,
// might not be the best node.
// so first we check if there is an error-node at the cursor-position
// @ts-ignore
const maybeErrorNode = getErrorNode(tree, pos);
const cur = maybeErrorNode != null ? maybeErrorNode.cursor() : tree.cursorAt(pos);
@ -561,10 +486,13 @@ export function getSituation(text: string, pos: number): Situation | null {
// i do not use a foreach because i want to stop as soon
// as i find something
if (isPathMatch(resolver.path, ids)) {
// @ts-ignore
return resolver.fun(currentNode, text, pos);
}
}
return null;
}
function notEmpty<TValue>(value: TValue | null | undefined): value is TValue {
return value !== null && value !== undefined;
}

21
packages/grafana-prometheus/src/querybuilder/parsing.test.ts
View File

@ -12,6 +12,7 @@ describe('buildVisualQueryFromString', () => {
})
);
});
it('parses simple binary comparison', () => {
expect(buildVisualQueryFromString('{app="aggregator"} == 11')).toEqual({
query: {
@ -56,6 +57,7 @@ describe('buildVisualQueryFromString', () => {
errors: [],
});
});
it('parses simple query', () => {
expect(buildVisualQueryFromString('counters_logins{app="frontend"}')).toEqual(
noErrors({
@ -87,6 +89,7 @@ describe('buildVisualQueryFromString', () => {
],
});
});
it('throws error when visual query parse with aggregation is ambiguous (scalar)', () => {
expect(buildVisualQueryFromString('topk(5, 1 / 2)')).toMatchObject({
errors: [
@ -98,6 +101,7 @@ describe('buildVisualQueryFromString', () => {
],
});
});
it('throws error when visual query parse with functionCall is ambiguous', () => {
expect(
buildVisualQueryFromString(
@ -113,6 +117,7 @@ describe('buildVisualQueryFromString', () => {
],
});
});
it('does not throw error when visual query parse is unambiguous', () => {
expect(
buildVisualQueryFromString('topk(5, node_arp_entries) / node_arp_entries{cluster="dev-eu-west-2"}')
@ -120,12 +125,14 @@ describe('buildVisualQueryFromString', () => {
errors: [],
});
});
it('does not throw error when visual query parse is unambiguous (scalar)', () => {
// Note this topk query with scalars is not valid in prometheus, but it does not currently throw an error during parse
expect(buildVisualQueryFromString('topk(5, 1) / 2')).toMatchObject({
errors: [],
});
});
it('does not throw error when visual query parse is unambiguous, function call', () => {
// Note this topk query with scalars is not valid in prometheus, but it does not currently throw an error during parse
expect(
@ -291,8 +298,7 @@ describe('buildVisualQueryFromString', () => {
});
});
// enable in #85942 when updated lezer parser is merged
xit('parses a native histogram function correctly', () => {
it('parses a native histogram function correctly', () => {
expect(
buildVisualQueryFromString('histogram_count(rate(counters_logins{app="backend"}[$__rate_interval]))')
).toEqual({
@ -306,7 +312,8 @@ describe('buildVisualQueryFromString', () => {
params: ['$__rate_interval'],
},
{
id: 'histogram_quantile',
id: 'histogram_count',
params: [],
},
],
},
@ -457,6 +464,12 @@ describe('buildVisualQueryFromString', () => {
to: 27,
parentType: 'VectorSelector',
},
{
text: ')',
from: 38,
to: 39,
parentType: 'PromQL',
},
],
query: {
metric: '${func_var}',
@ -710,7 +723,7 @@ describe('buildVisualQueryFromString', () => {
errors: [
{
from: 6,
parentType: 'Expr',
parentType: 'BinaryExpr',
text: '(bar + baz)',
to: 17,
},

80
packages/grafana-prometheus/src/querybuilder/parsing.ts
View File

@ -5,22 +5,18 @@ import {
AggregateModifier,
AggregateOp,
BinaryExpr,
BinModifiers,
Expr,
BoolModifier,
FunctionCall,
FunctionCallArgs,
FunctionCallBody,
FunctionIdentifier,
GroupingLabel,
GroupingLabelList,
GroupingLabels,
Identifier,
LabelMatcher,
LabelName,
MatchingModifierClause,
MatchOp,
MetricIdentifier,
NumberLiteral,
On,
OnOrIgnoring,
ParenExpr,
parser,
StringLiteral,
@ -102,6 +98,7 @@ interface Context {
errors: ParsingError[];
}
// TODO find a better approach for grafana global variables
function isValidPromQLMinusGrafanaGlobalVariables(expr: string) {
const context: Context = {
query: {
@ -142,7 +139,7 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex
const visQuery = context.query;
switch (node.type.id) {
case MetricIdentifier: {
case Identifier: {
// Expectation is that there is only one of those per query.
visQuery.metric = getString(expr, node);
break;
@ -183,8 +180,8 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex
default: {
if (node.type.id === ParenExpr) {
// We don't support parenthesis in the query to group expressions. We just report error but go on with the
// parsing.
// We don't support parenthesis in the query to group expressions.
// We just report error but go on with the parsing.
context.errors.push(makeError(expr, node));
}
// Any other nodes we just ignore and go to its children. This should be fine as there are lots of wrapper
@ -200,8 +197,9 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex
}
}
// TODO check if we still need this
function isIntervalVariableError(node: SyntaxNode) {
return node.prevSibling?.type.id === Expr && node.prevSibling?.firstChild?.type.id === VectorSelector;
return node.prevSibling?.firstChild?.type.id === VectorSelector;
}
function getLabel(expr: string, node: SyntaxNode): QueryBuilderLabelFilter {
@ -229,7 +227,6 @@ function handleFunction(expr: string, node: SyntaxNode, context: Context) {
const funcName = getString(expr, nameNode);
const body = node.getChild(FunctionCallBody);
const callArgs = body!.getChild(FunctionCallArgs);
const params = [];
let interval = '';
@ -249,13 +246,13 @@ function handleFunction(expr: string, node: SyntaxNode, context: Context) {
// We unshift operations to keep the more natural order that we want to have in the visual query editor.
visQuery.operations.unshift(op);
if (callArgs) {
if (getString(expr, callArgs) === interval + ']') {
if (body) {
if (getString(expr, body) === '([' + interval + '])') {
// This is a special case where we have a function with a single argument and it is the interval.
// This happens when you start adding operations in query builder and did not set a metric yet.
return;
}
updateFunctionArgs(expr, callArgs, context, op);
updateFunctionArgs(expr, body, context, op);
}
}
@ -284,25 +281,14 @@ function handleAggregation(expr: string, node: SyntaxNode, context: Context) {
funcName = `__${funcName}_without`;
}
labels.push(...getAllByType(expr, modifier, GroupingLabel));
labels.push(...getAllByType(expr, modifier, LabelName));
}
const body = node.getChild(FunctionCallBody);
const callArgs = body!.getChild(FunctionCallArgs);
const callArgsExprChild = callArgs?.getChild(Expr);
const binaryExpressionWithinAggregationArgs = callArgsExprChild?.getChild(BinaryExpr);
if (binaryExpressionWithinAggregationArgs) {
context.errors.push({
text: 'Query parsing is ambiguous.',
from: binaryExpressionWithinAggregationArgs.from,
to: binaryExpressionWithinAggregationArgs.to,
});
}
const op: QueryBuilderOperation = { id: funcName, params: [] };
visQuery.operations.unshift(op);
updateFunctionArgs(expr, callArgs, context, op);
updateFunctionArgs(expr, body, context, op);
// We add labels after params in the visual query editor.
op.params.push(...labels);
}
@ -310,8 +296,7 @@ function handleAggregation(expr: string, node: SyntaxNode, context: Context) {
/**
* Handle (probably) all types of arguments that function or aggregation can have.
*
* FunctionCallArgs are nested bit weirdly basically its [firstArg, ...rest] where rest is again FunctionCallArgs so
* we cannot just get all the children and iterate them as arguments we have to again recursively traverse through
* We cannot just get all the children and iterate them as arguments we have to again recursively traverse through
* them.
*
* @param expr
@ -324,15 +309,16 @@ function updateFunctionArgs(expr: string, node: SyntaxNode | null, context: Cont
return;
}
switch (node.type.id) {
// In case we have an expression we don't know what kind so we have to look at the child as it can be anything.
case Expr:
// FunctionCallArgs are nested bit weirdly as mentioned so we have to go one deeper in this case.
case FunctionCallArgs: {
case FunctionCallBody: {
let child = node.firstChild;
while (child) {
const callArgsExprChild = child.getChild(Expr);
const binaryExpressionWithinFunctionArgs = callArgsExprChild?.getChild(BinaryExpr);
let binaryExpressionWithinFunctionArgs: SyntaxNode | null;
if (child.type.id === BinaryExpr) {
binaryExpressionWithinFunctionArgs = child;
} else {
binaryExpressionWithinFunctionArgs = child.getChild(BinaryExpr);
}
if (binaryExpressionWithinFunctionArgs) {
context.errors.push({
@ -345,7 +331,6 @@ function updateFunctionArgs(expr: string, node: SyntaxNode | null, context: Cont
updateFunctionArgs(expr, child, context, op);
child = child.nextSibling;
}
break;
}
@ -378,16 +363,16 @@ function handleBinary(expr: string, node: SyntaxNode, context: Context) {
const visQuery = context.query;
const left = node.firstChild!;
const op = getString(expr, left.nextSibling);
const binModifier = getBinaryModifier(expr, node.getChild(BinModifiers));
const binModifier = getBinaryModifier(expr, node.getChild(BoolModifier) ?? node.getChild(MatchingModifierClause));
const right = node.lastChild!;
const opDef = binaryScalarOperatorToOperatorName[op];
const leftNumber = left.getChild(NumberLiteral);
const rightNumber = right.getChild(NumberLiteral);
const leftNumber = left.type.id === NumberLiteral;
const rightNumber = right.type.id === NumberLiteral;
const rightBinary = right.getChild(BinaryExpr);
const rightBinary = right.type.id === BinaryExpr;
if (leftNumber) {
// TODO: this should be already handled in case parent is binary expression as it has to be added to parent
@ -433,6 +418,7 @@ function handleBinary(expr: string, node: SyntaxNode, context: Context) {
}
}
// TODO revisit this function.
function getBinaryModifier(
expr: string,
node: SyntaxNode | null
@ -446,17 +432,17 @@ function getBinaryModifier(
if (node.getChild('Bool')) {
return { isBool: true, isMatcher: false };
} else {
const matcher = node.getChild(OnOrIgnoring);
if (!matcher) {
// Not sure what this could be, maybe should be an error.
return undefined;
let labels = '';
const groupingLabels = node.getChild(GroupingLabels);
if (groupingLabels) {
labels = getAllByType(expr, groupingLabels, LabelName).join(', ');
}
const labels = getString(expr, matcher.getChild(GroupingLabels)?.getChild(GroupingLabelList));
return {
isMatcher: true,
isBool: false,
matches: labels,
matchType: matcher.getChild(On) ? 'on' : 'ignoring',
matchType: node.getChild(On) ? 'on' : 'ignoring',
};
}
}

7
packages/grafana-prometheus/src/querybuilder/parsingUtils.ts
View File

@ -114,11 +114,10 @@ export function makeBinOp(
* not be safe is it would also find arguments of nested functions.
* @param expr
* @param cur
* @param type - can be string or number, some data-sources (loki) haven't migrated over to using numeric constants defined in the lezer parsing library (e.g. lezer-promql).
* @todo Remove string type definition when all data-sources have migrated to numeric constants
* @param type
*/
export function getAllByType(expr: string, cur: SyntaxNode, type: number | string): string[] {
if (cur.type.id === type || cur.name === type) {
export function getAllByType(expr: string, cur: SyntaxNode, type: number): string[] {
if (cur.type.id === type) {
return [getString(expr, cur)];
}
const values: string[] = [];

29
yarn.lock
View File

@ -4012,8 +4012,8 @@ __metadata:
"@leeoniya/ufuzzy": "npm:1.0.14"
"@lezer/common": "npm:1.2.1"
"@lezer/highlight": "npm:1.2.0"
"@lezer/lr": "npm:1.3.3"
"@prometheus-io/lezer-promql": "npm:^0.37.0-rc.1"
"@lezer/lr": "npm:1.4.0"
"@prometheus-io/lezer-promql": "npm:0.51.2"
"@reduxjs/toolkit": "npm:1.9.5"
"@rollup/plugin-image": "npm:3.0.3"
"@rollup/plugin-node-resolve": "npm:15.2.3"
@ -5041,6 +5041,15 @@ __metadata:
languageName: node
linkType: hard
"@lezer/lr@npm:1.4.0":
version: 1.4.0
resolution: "@lezer/lr@npm:1.4.0"
dependencies:
"@lezer/common": "npm:^1.0.0"
checksum: 10/7391d0d08e54cd9e4f4d46e6ee6aa81fbaf079b22ed9c13d01fc9928e0ffd16d0c2d21b2cedd55675ad6c687277db28349ea8db81c9c69222cd7e7c40edd026e
languageName: node
linkType: hard
"@linaria/core@npm:^4.5.4":
version: 4.5.4
resolution: "@linaria/core@npm:4.5.4"
@ -6152,13 +6161,23 @@ __metadata:
languageName: node
linkType: hard
"@prometheus-io/lezer-promql@npm:0.51.2":
version: 0.51.2
resolution: "@prometheus-io/lezer-promql@npm:0.51.2"
peerDependencies:
"@lezer/highlight": ^1.1.2
"@lezer/lr": ^1.2.3
checksum: 10/cee04e8bb24b54caa5da029ab66aade5245c8ed96a99ca2444b45a1a814dc03e01197e4b4d9dd767baa9f81c35441c879939e13517b5fd5854598ceb58087e6b
languageName: node
linkType: hard
"@prometheus-io/lezer-promql@npm:^0.37.0-rc.1":
version: 0.37.0
resolution: "@prometheus-io/lezer-promql@npm:0.37.0"
version: 0.37.9
resolution: "@prometheus-io/lezer-promql@npm:0.37.9"
peerDependencies:
"@lezer/highlight": ^1.0.0
"@lezer/lr": ^1.0.0
checksum: 10/00a3ef7a292ae17c7059da73e1ebd4568135eb5189be0eb60f039915f1c20a0bf355fe02cec1c11955e9e3885b5ecfdd8a67d57ce25fa09ad74575ba0fbc7386
checksum: 10/3b1ddd9b47e3ba4f016901d6fc1b3b7b75855fb5da568fb95b30bfc60d35065e89d64162d947312126163a314c8844fa4a72176f9babdf86c63837d3fc0a5e4a
languageName: node
linkType: hard