IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Prometheus: Fix expanding that contains multiple metrics (#82354)

Browse Source 
* fix expanding rules with one metric wrapped in a parenthesis

* fix expanding rules with regex match operator

* fix for multiple labels

* refactor

* don't modify recording rules name in label values

* metric + metric fix

* fix last issues with label regex and spaces

* add comments

* add the same changes to the prometheus library
This commit is contained in:
ismail simsek 2024-02-15 21:33:28 +01:00 committed by GitHub
parent c879588332
commit c540fd4195
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 275 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
packages/grafana-prometheus/src/language_utils.test.ts
View File

@ -167,10 +167,7 @@ describe('expandRecordingRules()', () => {
it('returns query with labels with expanded recording rules', () => {
expect(
expandRecordingRules('metricA{label1="value1"} / metricB{label2="value2"}', {
metricA: 'fooA',
metricB: 'fooB',
})
expandRecordingRules('metricA{label1="value1"} / metricB{label2="value2"}', { metricA: 'fooA', metricB: 'fooB' })
).toBe('fooA{label1="value1"} / fooB{label2="value2"}');
expect(
expandRecordingRules('metricA{label1="value1",label2="value,2"}', {
@ -182,13 +179,39 @@ describe('expandRecordingRules()', () => {
metricA: 'rate(fooA[])',
metricB: 'rate(fooB[])',
})
).toBe('rate(fooA{label1="value1"}[])/ rate(fooB{label2="value2"}[])');
).toBe('rate(fooA{label1="value1"}[]) / rate(fooB{label2="value2"}[])');
expect(
expandRecordingRules('metricA{label1="value1",label2="value2"} / metricB{label3="value3"}', {
metricA: 'rate(fooA[])',
metricB: 'rate(fooB[])',
})
).toBe('rate(fooA{label1="value1", label2="value2"}[])/ rate(fooB{label3="value3"}[])');
).toBe('rate(fooA{label1="value1", label2="value2"}[]) / rate(fooB{label3="value3"}[])');
});
it('expands the query even it is wrapped with parentheses', () => {
expect(
expandRecordingRules('sum (metric{label1="value1"}) by (env)', { metric: 'foo{labelInside="valueInside"}' })
).toBe('sum (foo{labelInside="valueInside", label1="value1"}) by (env)');
});
it('expands the query with regex match', () => {
expect(
expandRecordingRules('sum (metric{label1=~"/value1/(sa|sb)"}) by (env)', {
metric: 'foo{labelInside="valueInside"}',
})
).toBe('sum (foo{labelInside="valueInside", label1=~"/value1/(sa|sb)"}) by (env)');
});
it('ins:metric:per{pid="val-42", comp="api"}', () => {
const query = `aaa:111{pid="val-42", comp="api"} + bbb:222{pid="val-42"}`;
const mapping = {
'aaa:111':
'(max without (mp) (targetMetric{device=~"/dev/(sda1|sdb)"}) / max without (mp) (targetMetric2{device=~"/dev/(sda1|sdb)"}))',
'bbb:222': '(targetMetric2{device=~"/dev/(sda1|sdb)"})',
};
const expected = `(max without (mp) (targetMetric{device=~"/dev/(sda1|sdb)", pid="val-42", comp="api"}) / max without (mp) (targetMetric2{device=~"/dev/(sda1|sdb)", pid="val-42", comp="api"})) + (targetMetric2{device=~"/dev/(sda1|sdb)", pid="val-42"})`;
const result = expandRecordingRules(query, mapping);
expect(result).toBe(expected);
});
});

124
packages/grafana-prometheus/src/language_utils.ts
View File

@ -64,7 +64,15 @@ export function processLabels(labels: Array<{ [key: string]: string }>, withName
// const cleanSelectorRegexp = /\{(\w+="[^"\n]*?")(,\w+="[^"\n]*?")*\}/;
export const selectorRegexp = /\{[^}]*?(\}|$)/;
export const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g;
// This will capture 4 groups. Example label filter => {instance="10.4.11.4:9003"}
// 1. label: instance
// 2. operator: =
// 3. value: "10.4.11.4:9003"
// 4. comma: if there is a comma it will give ,
// 5. space: if there is a space after comma it will give the whole space
// comma and space is useful for addLabelsToExpression function
export const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")(,)?(\s*)?/g;
export function parseSelector(query: string, cursorOffset = 1): { labelKeys: any[]; selector: string } {
if (!query.match(selectorRegexp)) {
@ -131,20 +139,88 @@ export function parseSelector(query: string, cursorOffset = 1): { labelKeys: any
}
export function expandRecordingRules(query: string, mapping: { [name: string]: string }): string {
const ruleNames = Object.keys(mapping);
const rulesRegex = new RegExp(`(\\s|^)(${ruleNames.join('|')})(\\s|$|\\(|\\[|\\{)`, 'ig');
const expandedQuery = query.replace(rulesRegex, (match, pre, name, post) => `${pre}${mapping[name]}${post}`);
const getRuleRegex = (ruleName: string) => new RegExp(`(\\s|\\(|^)(${ruleName})(\\s|$|\\(|\\[|\\{)`, 'ig');
// Split query into array, so if query uses operators, we can correctly add labels to each individual part.
const queryArray = expandedQuery.split(/(\+|\-|\*|\/|\%|\^)/);
// For each mapping key we iterate over the query and split them in parts.
// recording:rule{label=~"/label/value"} * some:other:rule{other_label="value"}
// We want to keep parts in here like this:
// recording:rule
// {label=~"/label/value"} *
// some:other:rule
// {other_label="value"}
const tmpSplitParts = Object.keys(mapping).reduce<string[]>(
(prev, curr) => {
let parts: string[] = [];
let tmpParts: string[] = [];
let removeIdx: number[] = [];
// Regex that matches occurrences of ){ or }{ or ]{ which is a sign of incorrecly added labels.
const invalidLabelsRegex = /(\)\{|\}\{|\]\{)/;
const correctlyExpandedQueryArray = queryArray.map((query) => {
return addLabelsToExpression(query, invalidLabelsRegex);
// we iterate over prev because it might be like this after first loop
// recording:rule and {label=~"/label/value"} * some:other:rule{other_label="value"}
// so we need to split the second part too
prev.filter(Boolean).forEach((p, i) => {
const doesMatch = p.match(getRuleRegex(curr));
if (doesMatch) {
parts = p.split(curr);
if (parts.length === 2) {
// this is the case when we have such result for this query
// max (metric{label="value"})
// "max(", "{label="value"}"
removeIdx.push(i);
tmpParts.push(...[parts[0], curr, parts[1]].filter(Boolean));
} else if (parts.length > 2) {
// this is the case when we have such query
// metric + metric
// when we split it we have such data
// "", " + ", ""
removeIdx.push(i);
parts = parts.map((p) => (p === '' ? curr : p));
tmpParts.push(...parts);
}
}
});
// if we have idx to remove that means we split the value in that index.
// No need to keep it. Have the new split values instead.
removeIdx.forEach((ri) => (prev[ri] = ''));
prev = prev.filter(Boolean);
prev.push(...tmpParts);
return prev;
},
[query]
);
// we have the separate parts. we need to replace the metric and apply the labels if there is any
let labelFound = false;
const trulyExpandedQuery = tmpSplitParts.map((tsp, i) => {
// if we know this loop tsp is a label, not the metric we want to expand
if (labelFound) {
labelFound = false;
return '';
}
// check if the mapping is there
if (mapping[tsp]) {
const recordingRule = mapping[tsp];
// it is a recording rule. if the following is a label then apply it
if (i + 1 !== tmpSplitParts.length && tmpSplitParts[i + 1].match(labelRegexp)) {
// the next value in the loop is label. Let's apply labels to the metric
labelFound = true;
const labels = tmpSplitParts[i + 1];
const invalidLabelsRegex = /(\)\{|\}\{|\]\{)/;
return addLabelsToExpression(recordingRule + labels, invalidLabelsRegex);
} else {
// it is not a recording rule and might be a binary operation in between two recording rules
// So no need to do anything. just return it.
return recordingRule;
}
}
return tsp;
});
return correctlyExpandedQueryArray.join('');
// Remove empty strings and merge them
return trulyExpandedQuery.filter(Boolean).join('');
}
function addLabelsToExpression(expr: string, invalidLabelsRegexp: RegExp) {
@ -159,9 +235,15 @@ function addLabelsToExpression(expr: string, invalidLabelsRegexp: RegExp) {
const exprAfterRegexMatch = expr.slice(indexOfRegexMatch + 1);
// Create arrayOfLabelObjects with label objects that have key, operator and value.
const arrayOfLabelObjects: Array<{ key: string; operator: string; value: string }> = [];
exprAfterRegexMatch.replace(labelRegexp, (label, key, operator, value) => {
arrayOfLabelObjects.push({ key, operator, value });
const arrayOfLabelObjects: Array<{
key: string;
operator: string;
value: string;
comma?: string;
space?: string;
}> = [];
exprAfterRegexMatch.replace(labelRegexp, (label, key, operator, value, comma, space) => {
arrayOfLabelObjects.push({ key, operator, value, comma, space });
return '';
});
@ -174,7 +256,19 @@ function addLabelsToExpression(expr: string, invalidLabelsRegexp: RegExp) {
result = addLabelToQuery(result, obj.key, value, obj.operator);
});
return result;
// reconstruct the labels
let existingLabel = arrayOfLabelObjects.reduce((prev, curr) => {
prev += `${curr.key}${curr.operator}${curr.value}${curr.comma ?? ''}${curr.space ?? ''}`;
return prev;
}, '');
// Check if there is anything besides labels
// Useful for this kind of metrics sum (recording_rule_metric{label1="value1"}) by (env)
// if we don't check this part, ) by (env) part will be lost
existingLabel = '{' + existingLabel + '}';
const potentialLeftOver = exprAfterRegexMatch.replace(existingLabel, '');
return result + potentialLeftOver;
}
/**

30
public/app/plugins/datasource/prometheus/language_utils.test.ts
View File

@ -176,13 +176,39 @@ describe('expandRecordingRules()', () => {
metricA: 'rate(fooA[])',
metricB: 'rate(fooB[])',
})
).toBe('rate(fooA{label1="value1"}[])/ rate(fooB{label2="value2"}[])');
).toBe('rate(fooA{label1="value1"}[]) / rate(fooB{label2="value2"}[])');
expect(
expandRecordingRules('metricA{label1="value1",label2="value2"} / metricB{label3="value3"}', {
metricA: 'rate(fooA[])',
metricB: 'rate(fooB[])',
})
).toBe('rate(fooA{label1="value1", label2="value2"}[])/ rate(fooB{label3="value3"}[])');
).toBe('rate(fooA{label1="value1", label2="value2"}[]) / rate(fooB{label3="value3"}[])');
});
it('expands the query even it is wrapped with parentheses', () => {
expect(
expandRecordingRules('sum (metric{label1="value1"}) by (env)', { metric: 'foo{labelInside="valueInside"}' })
).toBe('sum (foo{labelInside="valueInside", label1="value1"}) by (env)');
});
it('expands the query with regex match', () => {
expect(
expandRecordingRules('sum (metric{label1=~"/value1/(sa|sb)"}) by (env)', {
metric: 'foo{labelInside="valueInside"}',
})
).toBe('sum (foo{labelInside="valueInside", label1=~"/value1/(sa|sb)"}) by (env)');
});
it('ins:metric:per{pid="val-42", comp="api"}', () => {
const query = `aaa:111{pid="val-42", comp="api"} + bbb:222{pid="val-42"}`;
const mapping = {
'aaa:111':
'(max without (mp) (targetMetric{device=~"/dev/(sda1|sdb)"}) / max without (mp) (targetMetric2{device=~"/dev/(sda1|sdb)"}))',
'bbb:222': '(targetMetric2{device=~"/dev/(sda1|sdb)"})',
};
const expected = `(max without (mp) (targetMetric{device=~"/dev/(sda1|sdb)", pid="val-42", comp="api"}) / max without (mp) (targetMetric2{device=~"/dev/(sda1|sdb)", pid="val-42", comp="api"})) + (targetMetric2{device=~"/dev/(sda1|sdb)", pid="val-42"})`;
const result = expandRecordingRules(query, mapping);
expect(result).toBe(expected);
});
});

124
public/app/plugins/datasource/prometheus/language_utils.ts
View File

@ -64,7 +64,15 @@ export function processLabels(labels: Array<{ [key: string]: string }>, withName
// const cleanSelectorRegexp = /\{(\w+="[^"\n]*?")(,\w+="[^"\n]*?")*\}/;
export const selectorRegexp = /\{[^}]*?(\}|$)/;
export const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g;
// This will capture 4 groups. Example label filter => {instance="10.4.11.4:9003"}
// 1. label: instance
// 2. operator: =
// 3. value: "10.4.11.4:9003"
// 4. comma: if there is a comma it will give ,
// 5. space: if there is a space after comma it will give the whole space
// comma and space is useful for addLabelsToExpression function
export const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")(,)?(\s*)?/g;
export function parseSelector(query: string, cursorOffset = 1): { labelKeys: any[]; selector: string } {
if (!query.match(selectorRegexp)) {
@ -131,20 +139,88 @@ export function parseSelector(query: string, cursorOffset = 1): { labelKeys: any
}
export function expandRecordingRules(query: string, mapping: { [name: string]: string }): string {
const ruleNames = Object.keys(mapping);
const rulesRegex = new RegExp(`(\\s|^)(${ruleNames.join('|')})(\\s|$|\\(|\\[|\\{)`, 'ig');
const expandedQuery = query.replace(rulesRegex, (match, pre, name, post) => `${pre}${mapping[name]}${post}`);
const getRuleRegex = (ruleName: string) => new RegExp(`(\\s|\\(|^)(${ruleName})(\\s|$|\\(|\\[|\\{)`, 'ig');
// Split query into array, so if query uses operators, we can correctly add labels to each individual part.
const queryArray = expandedQuery.split(/(\+|\-|\*|\/|\%|\^)/);
// For each mapping key we iterate over the query and split them in parts.
// recording:rule{label=~"/label/value"} * some:other:rule{other_label="value"}
// We want to keep parts in here like this:
// recording:rule
// {label=~"/label/value"} *
// some:other:rule
// {other_label="value"}
const tmpSplitParts = Object.keys(mapping).reduce<string[]>(
(prev, curr) => {
let parts: string[] = [];
let tmpParts: string[] = [];
let removeIdx: number[] = [];
// Regex that matches occurrences of ){ or }{ or ]{ which is a sign of incorrecly added labels.
const invalidLabelsRegex = /(\)\{|\}\{|\]\{)/;
const correctlyExpandedQueryArray = queryArray.map((query) => {
return addLabelsToExpression(query, invalidLabelsRegex);
// we iterate over prev because it might be like this after first loop
// recording:rule and {label=~"/label/value"} * some:other:rule{other_label="value"}
// so we need to split the second part too
prev.filter(Boolean).forEach((p, i) => {
const doesMatch = p.match(getRuleRegex(curr));
if (doesMatch) {
parts = p.split(curr);
if (parts.length === 2) {
// this is the case when we have such result for this query
// max (metric{label="value"})
// "max(", "{label="value"}"
removeIdx.push(i);
tmpParts.push(...[parts[0], curr, parts[1]].filter(Boolean));
} else if (parts.length > 2) {
// this is the case when we have such query
// metric + metric
// when we split it we have such data
// "", " + ", ""
removeIdx.push(i);
parts = parts.map((p) => (p === '' ? curr : p));
tmpParts.push(...parts);
}
}
});
// if we have idx to remove that means we split the value in that index.
// No need to keep it. Have the new split values instead.
removeIdx.forEach((ri) => (prev[ri] = ''));
prev = prev.filter(Boolean);
prev.push(...tmpParts);
return prev;
},
[query]
);
// we have the separate parts. we need to replace the metric and apply the labels if there is any
let labelFound = false;
const trulyExpandedQuery = tmpSplitParts.map((tsp, i) => {
// if we know this loop tsp is a label, not the metric we want to expand
if (labelFound) {
labelFound = false;
return '';
}
// check if the mapping is there
if (mapping[tsp]) {
const recordingRule = mapping[tsp];
// it is a recording rule. if the following is a label then apply it
if (i + 1 !== tmpSplitParts.length && tmpSplitParts[i + 1].match(labelRegexp)) {
// the next value in the loop is label. Let's apply labels to the metric
labelFound = true;
const labels = tmpSplitParts[i + 1];
const invalidLabelsRegex = /(\)\{|\}\{|\]\{)/;
return addLabelsToExpression(recordingRule + labels, invalidLabelsRegex);
} else {
// it is not a recording rule and might be a binary operation in between two recording rules
// So no need to do anything. just return it.
return recordingRule;
}
}
return tsp;
});
return correctlyExpandedQueryArray.join('');
// Remove empty strings and merge them
return trulyExpandedQuery.filter(Boolean).join('');
}
function addLabelsToExpression(expr: string, invalidLabelsRegexp: RegExp) {
@ -159,9 +235,15 @@ function addLabelsToExpression(expr: string, invalidLabelsRegexp: RegExp) {
const exprAfterRegexMatch = expr.slice(indexOfRegexMatch + 1);
// Create arrayOfLabelObjects with label objects that have key, operator and value.
const arrayOfLabelObjects: Array<{ key: string; operator: string; value: string }> = [];
exprAfterRegexMatch.replace(labelRegexp, (label, key, operator, value) => {
arrayOfLabelObjects.push({ key, operator, value });
const arrayOfLabelObjects: Array<{
key: string;
operator: string;
value: string;
comma?: string;
space?: string;
}> = [];
exprAfterRegexMatch.replace(labelRegexp, (label, key, operator, value, comma, space) => {
arrayOfLabelObjects.push({ key, operator, value, comma, space });
return '';
});
@ -174,7 +256,19 @@ function addLabelsToExpression(expr: string, invalidLabelsRegexp: RegExp) {
result = addLabelToQuery(result, obj.key, value, obj.operator);
});
return result;
// reconstruct the labels
let existingLabel = arrayOfLabelObjects.reduce((prev, curr) => {
prev += `${curr.key}${curr.operator}${curr.value}${curr.comma ?? ''}${curr.space ?? ''}`;
return prev;
}, '');
// Check if there is anything besides labels
// Useful for this kind of metrics sum (recording_rule_metric{label1="value1"}) by (env)
// if we don't check this part, ) by (env) part will be lost
existingLabel = '{' + existingLabel + '}';
const potentialLeftOver = exprAfterRegexMatch.replace(existingLabel, '');
return result + potentialLeftOver;
}
/**