IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-11 16:15:42 -06:00
Code Issues Packages Projects Releases Wiki Activity

Loki Query Editor: Autocompletion and suggestions improvements (unwrap, parser, extracted labels) (#59103)

Browse Source 
* Chore: refactor test to improve internal categorization of scenarios

* feat(loki-monaco-unwrap): add unwrap situation support

* Chore: remove redundant path from aggregation situation

* feat(loki-monaco-unwrap): add unwrap suggestions

* feat(loki-monaco-autocomplete): rename IN_DURATION and add missing tests

* feat(loki-monaco-autocomplete): detect parser and line filter

* feat(loki-monaco-autocomplete): optionally suggest line filters and parsers

* feat(loki-monaco-autocomplete): improve suggestions and update completions tests

* feat(loki-monaco-autocomplete): allow for multiple line filters suggestions

* Chore: update situations test

* Chore: add test case for AFTER_UNWRAP completion

* feat(loki-monaco-autocomplete): use logs query instead of labels for data sample

* Chore: improve getParser function name and add tests

* Chore: update test mock data

* Update public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.test.ts

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>

* feat(loki-monaco-autocomplete): improve after unwrap detection

* feat(loki-monaco-autocomplete): remove leftover parser detection

* Chore: completely remove parser suggestion exclusion implementation

It was correct to have more than one parser, so we don't need to exclude parsers if one is present.

Co-authored-by: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com>
This commit is contained in:
Matias Chomicki 2022-12-14 17:37:08 +01:00 committed by GitHub
parent 3605103e28
commit 4be99c56f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 227 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/CompletionDataProvider.test.ts
View File

@ -88,7 +88,7 @@ describe('CompletionDataProvider', () => {
});
test('Returns the expected parser and label keys', async () => {
expect(await completionProvider.getParserAndLabelKeys([])).toEqual(parserAndLabelKeys);
expect(await completionProvider.getParserAndLabelKeys('')).toEqual(parserAndLabelKeys);
});
test('Returns the expected series labels', async () => {

4
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/CompletionDataProvider.ts
View File

@ -55,8 +55,8 @@ export class CompletionDataProvider {
return data[labelName] ?? [];
}
async getParserAndLabelKeys(labels: Label[]) {
return await this.languageProvider.getParserAndLabelKeys(this.buildSelector(labels));
async getParserAndLabelKeys(logQuery: string) {
return await this.languageProvider.getParserAndLabelKeys(logQuery);
}
async getSeriesLabels(labels: Label[]) {

44
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.test.ts
View File

@ -100,17 +100,17 @@ const afterSelectorCompletions = [
{
insertText: '| unwrap extracted',
label: 'unwrap extracted',
type: 'LINE_FILTER',
type: 'PIPE_OPERATION',
},
{
insertText: '| unwrap place',
label: 'unwrap place',
type: 'LINE_FILTER',
type: 'PIPE_OPERATION',
},
{
insertText: '| unwrap source',
label: 'unwrap source',
type: 'LINE_FILTER',
type: 'PIPE_OPERATION',
},
{
insertText: '| unwrap',
@ -134,18 +134,13 @@ const afterSelectorCompletions = [
},
];
function buildAfterSelectorCompletions(
detectedParser: string,
detectedParserType: string,
otherParser: string,
afterPipe: boolean
) {
function buildAfterSelectorCompletions(detectedParser: string, otherParser: string, afterPipe: boolean) {
const explanation = '(detected)';
const expectedCompletions = afterSelectorCompletions.map((completion) => {
if (completion.type === 'DETECTED_PARSER_PLACEHOLDER') {
return {
...completion,
type: detectedParserType,
type: 'PARSER',
label: `${detectedParser} ${explanation}`,
insertText: `| ${detectedParser}`,
};
@ -200,8 +195,8 @@ describe('getCompletions', () => {
expect(completions).toHaveLength(24);
});
test('Returns completion options when the situation is IN_DURATION', async () => {
const situation: Situation = { type: 'IN_DURATION' };
test('Returns completion options when the situation is IN_RANGE', async () => {
const situation: Situation = { type: 'IN_RANGE' };
const completions = await getCompletions(situation, completionProvider);
expect(completions).toEqual([
@ -217,7 +212,7 @@ describe('getCompletions', () => {
});
test('Returns completion options when the situation is IN_GROUPING', async () => {
const situation: Situation = { type: 'IN_GROUPING', otherLabels };
const situation: Situation = { type: 'IN_GROUPING', logQuery: '' };
const completions = await getCompletions(situation, completionProvider);
expect(completions).toEqual([
@ -311,33 +306,33 @@ describe('getCompletions', () => {
});
test.each([true, false])(
'Returns completion options when the situation is AFTER_SELECTOR, JSON parser, and afterPipe %s',
'Returns completion options when the situation is AFTER_SELECTOR, detected JSON parser, and afterPipe %s',
async (afterPipe: boolean) => {
jest.spyOn(completionProvider, 'getParserAndLabelKeys').mockResolvedValue({
extractedLabelKeys,
hasJSON: true,
hasLogfmt: false,
});
const situation: Situation = { type: 'AFTER_SELECTOR', labels: [], afterPipe };
const situation: Situation = { type: 'AFTER_SELECTOR', logQuery: '', afterPipe };
const completions = await getCompletions(situation, completionProvider);
const expected = buildAfterSelectorCompletions('json', 'PARSER', 'logfmt', afterPipe);
const expected = buildAfterSelectorCompletions('json', 'logfmt', afterPipe);
expect(completions).toEqual(expected);
}
);
test.each([true, false])(
'Returns completion options when the situation is AFTER_SELECTOR, Logfmt parser, and afterPipe %s',
'Returns completion options when the situation is AFTER_SELECTOR, detected Logfmt parser, and afterPipe %s',
async (afterPipe: boolean) => {
jest.spyOn(completionProvider, 'getParserAndLabelKeys').mockResolvedValue({
extractedLabelKeys,
hasJSON: false,
hasLogfmt: true,
});
const situation: Situation = { type: 'AFTER_SELECTOR', labels: [], afterPipe };
const situation: Situation = { type: 'AFTER_SELECTOR', logQuery: '', afterPipe };
const completions = await getCompletions(situation, completionProvider);
const expected = buildAfterSelectorCompletions('logfmt', 'DURATION', 'json', afterPipe);
const expected = buildAfterSelectorCompletions('logfmt', 'json', afterPipe);
expect(completions).toEqual(expected);
}
);
@ -348,4 +343,15 @@ describe('getCompletions', () => {
expect(completions).toHaveLength(22);
});
test('Returns completion options when the situation is AFTER_UNWRAP', async () => {
const situation: Situation = { type: 'AFTER_UNWRAP', logQuery: '' };
const completions = await getCompletions(situation, completionProvider);
const extractedCompletions = completions.filter((completion) => completion.type === 'LABEL_NAME');
const functionCompletions = completions.filter((completion) => completion.type === 'FUNCTION');
expect(extractedCompletions).toHaveLength(3);
expect(functionCompletions).toHaveLength(3);
});
});

85
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.ts
View File

@ -66,6 +66,27 @@ const DURATION_COMPLETIONS: Completion[] = ['$__interval', '$__range', '1m', '5m
})
);
const UNWRAP_FUNCTION_COMPLETIONS: Completion[] = [
{
type: 'FUNCTION',
label: 'duration_seconds',
documentation: 'Will convert the label value in seconds from the go duration format (e.g 5m, 24s30ms).',
insertText: 'duration_seconds()',
},
{
type: 'FUNCTION',
label: 'duration',
documentation: 'Short version of duration_seconds().',
insertText: 'duration()',
},
{
type: 'FUNCTION',
label: 'bytes',
documentation: 'Will convert the label value to raw bytes applying the bytes unit (e.g. 5 MiB, 3k, 1G).',
insertText: 'bytes()',
},
];
const LINE_FILTER_COMPLETIONS = [
{
operator: '|=',
@ -123,11 +144,8 @@ async function getLabelNamesForSelectorCompletions(
}));
}
async function getInGroupingCompletions(
otherLabels: Label[],
dataProvider: CompletionDataProvider
): Promise<Completion[]> {
const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(otherLabels);
async function getInGroupingCompletions(logQuery: string, dataProvider: CompletionDataProvider): Promise<Completion[]> {
const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery);
return extractedLabelKeys.map((label) => ({
type: 'LABEL_NAME',
@ -139,16 +157,17 @@ async function getInGroupingCompletions(
const PARSERS = ['json', 'logfmt', 'pattern', 'regexp', 'unpack'];
async function getAfterSelectorCompletions(
labels: Label[],
async function getParserCompletions(
afterPipe: boolean,
dataProvider: CompletionDataProvider
): Promise<Completion[]> {
const { extractedLabelKeys, hasJSON, hasLogfmt } = await dataProvider.getParserAndLabelKeys(labels);
hasJSON: boolean,
hasLogfmt: boolean,
extractedLabelKeys: string[]
) {
const allParsers = new Set(PARSERS);
const completions: Completion[] = [];
const prefix = afterPipe ? ' ' : '| ';
const hasLevelInExtractedLabels = extractedLabelKeys.some((key) => key === 'level');
if (hasJSON) {
allParsers.delete('json');
const extra = hasLevelInExtractedLabels ? '' : ' (detected)';
@ -166,7 +185,7 @@ async function getAfterSelectorCompletions(
allParsers.delete('logfmt');
const extra = hasLevelInExtractedLabels ? '' : ' (detected)';
completions.push({
type: 'DURATION',
type: 'PARSER',
label: `logfmt${extra}`,
insertText: `${prefix}logfmt`,
documentation: hasLevelInExtractedLabels
@ -185,9 +204,23 @@ async function getAfterSelectorCompletions(
});
});
return completions;
}
async function getAfterSelectorCompletions(
logQuery: string,
afterPipe: boolean,
dataProvider: CompletionDataProvider
): Promise<Completion[]> {
const { extractedLabelKeys, hasJSON, hasLogfmt } = await dataProvider.getParserAndLabelKeys(logQuery);
const completions: Completion[] = await getParserCompletions(afterPipe, hasJSON, hasLogfmt, extractedLabelKeys);
const prefix = afterPipe ? ' ' : '| ';
extractedLabelKeys.forEach((key) => {
completions.push({
type: 'LINE_FILTER',
type: 'PIPE_OPERATION',
label: `unwrap ${key}`,
insertText: `${prefix}unwrap ${key}`,
});
@ -216,7 +249,9 @@ async function getAfterSelectorCompletions(
documentation: explainOperator(LokiOperationId.LabelFormat),
});
return [...getLineFilterCompletions(afterPipe), ...completions];
const lineFilters = getLineFilterCompletions(afterPipe);
return [...lineFilters, ...completions];
}
async function getLabelValuesForMetricCompletions(
@ -233,6 +268,22 @@ async function getLabelValuesForMetricCompletions(
}));
}
async function getAfterUnwrapCompletions(
logQuery: string,
dataProvider: CompletionDataProvider
): Promise<Completion[]> {
const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery);
const labelCompletions: Completion[] = extractedLabelKeys.map((label) => ({
type: 'LABEL_NAME',
label,
insertText: label,
triggerOnInsert: false,
}));
return [...labelCompletions, ...UNWRAP_FUNCTION_COMPLETIONS];
}
export async function getCompletions(
situation: Situation,
dataProvider: CompletionDataProvider
@ -242,10 +293,10 @@ export async function getCompletions(
case 'AT_ROOT':
const historyCompletions = await getAllHistoryCompletions(dataProvider);
return [...historyCompletions, ...LOG_COMPLETIONS, ...AGGREGATION_COMPLETIONS, ...FUNCTION_COMPLETIONS];
case 'IN_DURATION':
case 'IN_RANGE':
return DURATION_COMPLETIONS;
case 'IN_GROUPING':
return getInGroupingCompletions(situation.otherLabels, dataProvider);
return getInGroupingCompletions(situation.logQuery, dataProvider);
case 'IN_LABEL_SELECTOR_NO_LABEL_NAME':
return getLabelNamesForSelectorCompletions(situation.otherLabels, dataProvider);
case 'IN_LABEL_SELECTOR_WITH_LABEL_NAME':
@ -256,7 +307,9 @@ export async function getCompletions(
dataProvider
);
case 'AFTER_SELECTOR':
return getAfterSelectorCompletions(situation.labels, situation.afterPipe, dataProvider);
return getAfterSelectorCompletions(situation.logQuery, situation.afterPipe, dataProvider);
case 'AFTER_UNWRAP':
return getAfterUnwrapCompletions(situation.logQuery, dataProvider);
case 'IN_AGGREGATION':
return [...FUNCTION_COMPLETIONS, ...AGGREGATION_COMPLETIONS];
default:

72
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.test.ts
View File

@ -26,19 +26,23 @@ function assertSituation(situation: string, expectedSituation: Situation | null)
}
describe('situation', () => {
it('handles things', () => {
it('identifies EMPTY autocomplete situations', () => {
assertSituation('^', {
type: 'EMPTY',
});
});
it('identifies EMPTY autocomplete situations', () => {
assertSituation('s^', {
type: 'AT_ROOT',
});
});
it('identifies AFTER_SELECTOR autocomplete situations', () => {
assertSituation('{level="info"} ^', {
type: 'AFTER_SELECTOR',
afterPipe: false,
labels: [{ name: 'level', value: 'info', op: '=' }],
logQuery: '{level="info"}',
});
// should not trigger AFTER_SELECTOR before the selector
@ -50,19 +54,19 @@ describe('situation', () => {
assertSituation('{level="info"} | json ^', {
type: 'AFTER_SELECTOR',
afterPipe: false,
labels: [{ name: 'level', value: 'info', op: '=' }],
logQuery: '{level="info"} | json',
});
assertSituation('{level="info"} | json | ^', {
type: 'AFTER_SELECTOR',
afterPipe: true,
labels: [{ name: 'level', value: 'info', op: '=' }],
logQuery: '{level="info"} | json |',
});
assertSituation('count_over_time({level="info"}^[10s])', {
type: 'AFTER_SELECTOR',
afterPipe: false,
labels: [{ name: 'level', value: 'info', op: '=' }],
logQuery: '{level="info"}',
});
// should not trigger AFTER_SELECTOR before the selector
@ -72,28 +76,49 @@ describe('situation', () => {
assertSituation('count_over_time({level="info"}^)', {
type: 'AFTER_SELECTOR',
afterPipe: false,
labels: [{ name: 'level', value: 'info', op: '=' }],
logQuery: '{level="info"}',
});
assertSituation('{level="info"} |= "a" | logfmt ^', {
type: 'AFTER_SELECTOR',
afterPipe: false,
logQuery: '{level="info"} |= "a" | logfmt',
});
assertSituation('sum(count_over_time({place="luna"} | logfmt |^)) by (place)', {
type: 'AFTER_SELECTOR',
afterPipe: true,
logQuery: '{place="luna"}| logfmt |',
});
});
it('identifies IN_AGGREGATION autocomplete situations', () => {
assertSituation('sum(^)', {
type: 'IN_AGGREGATION',
});
});
it('handles label names', () => {
it('identifies IN_LABEL_SELECTOR_NO_LABEL_NAME autocomplete situations', () => {
assertSituation('{^}', {
type: 'IN_LABEL_SELECTOR_NO_LABEL_NAME',
otherLabels: [],
});
assertSituation('sum({^})', {
type: 'IN_LABEL_SELECTOR_NO_LABEL_NAME',
otherLabels: [],
});
});
it('identifies labels from queries', () => {
assertSituation('sum(count_over_time({level="info"})) by (^)', {
type: 'IN_GROUPING',
otherLabels: [{ name: 'level', value: 'info', op: '=' }],
logQuery: '{level="info"}',
});
assertSituation('sum by (^) (count_over_time({level="info"}))', {
type: 'IN_GROUPING',
otherLabels: [{ name: 'level', value: 'info', op: '=' }],
logQuery: '{level="info"}',
});
assertSituation('{one="val1",two!="val2",three=~"val3",four!~"val4",^}', {
@ -124,6 +149,35 @@ describe('situation', () => {
});
});
it('identifies AFTER_UNWRAP autocomplete situations', () => {
assertSituation('sum(sum_over_time({one="val1"} | unwrap^', {
type: 'AFTER_UNWRAP',
logQuery: '{one="val1"}',
});
assertSituation(
'quantile_over_time(0.99, {cluster="ops-tools1",container="ingress-nginx"} | json | __error__ = "" | unwrap ^',
{
type: 'AFTER_UNWRAP',
logQuery: '{cluster="ops-tools1",container="ingress-nginx"}| json | __error__ = ""',
}
);
assertSituation('sum(sum_over_time({place="luna"} | unwrap ^ [5m])) by (level)', {
type: 'AFTER_UNWRAP',
logQuery: '{place="luna"}',
});
});
it.each(['count_over_time({job="mysql"}[^])', 'rate({instance="server\\1"}[^])', 'rate({}[^'])(
'identifies IN_RANGE autocomplete situations in metric query %s',
(query: string) => {
assertSituation(query, {
type: 'IN_RANGE',
});
}
);
it('handles label values', () => {
assertSituation('{job=^}', {
type: 'IN_LABEL_SELECTOR_WITH_LABEL_NAME',

57
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.ts
View File

@ -19,8 +19,11 @@ import {
Expr,
LiteralExpr,
MetricExpr,
UnwrapExpr,
} from '@grafana/lezer-logql';
import { getLogQueryFromMetricsQuery } from '../../../queryUtils';
type Direction = 'parent' | 'firstChild' | 'lastChild' | 'nextSibling';
type NodeType = number;
@ -94,14 +97,14 @@ export type Situation =
type: 'AT_ROOT';
}
| {
type: 'IN_DURATION';
type: 'IN_RANGE';
}
| {
type: 'IN_AGGREGATION';
}
| {
type: 'IN_GROUPING';
otherLabels: Label[];
logQuery: string;
}
| {
type: 'IN_LABEL_SELECTOR_NO_LABEL_NAME';
@ -116,7 +119,11 @@ export type Situation =
| {
type: 'AFTER_SELECTOR';
afterPipe: boolean;
labels: Label[];
logQuery: string;
}
| {
type: 'AFTER_UNWRAP';
logQuery: string;
};
type Resolver = {
@ -164,13 +171,21 @@ const RESOLVERS: Resolver[] = [
fun: resolveLogRangeFromError,
},
{
path: [ERROR_NODE_ID, LiteralExpr, MetricExpr, VectorAggregationExpr, MetricExpr, Expr, LogQL],
path: [ERROR_NODE_ID, LiteralExpr, MetricExpr, VectorAggregationExpr],
fun: () => ({ type: 'IN_AGGREGATION' }),
},
{
path: [ERROR_NODE_ID, PipelineStage, PipelineExpr],
fun: resolvePipeError,
},
{
path: [ERROR_NODE_ID, UnwrapExpr],
fun: resolveAfterUnwrap,
},
{
path: [UnwrapExpr],
fun: resolveAfterUnwrap,
},
];
const LABEL_OP_MAP = new Map<string, LabelOperator>([
@ -248,6 +263,13 @@ function getLabels(selectorNode: SyntaxNode, text: string): Label[] {
return labels;
}
function resolveAfterUnwrap(node: SyntaxNode, text: string, pos: number): Situation | null {
return {
type: 'AFTER_UNWRAP',
logQuery: getLogQueryFromMetricsQuery(text).trim(),
};
}
function resolvePipeError(node: SyntaxNode, text: string, pos: number): Situation | null {
// for example `{level="info"} |`
const exprNode = walk(node, [
@ -292,11 +314,9 @@ function resolveLabelsForGrouping(node: SyntaxNode, text: string, pos: number):
return null;
}
const otherLabels = getLabels(selectorNode, text);
return {
type: 'IN_GROUPING',
otherLabels,
logQuery: getLogQueryFromMetricsQuery(text).trim(),
};
}
@ -400,7 +420,7 @@ function resolveTopLevel(node: SyntaxNode, text: string, pos: number): Situation
function resolveDurations(node: SyntaxNode, text: string, pos: number): Situation {
return {
type: 'IN_DURATION',
type: 'IN_RANGE',
};
}
@ -418,21 +438,20 @@ function resolveLogRangeFromError(node: SyntaxNode, text: string, pos: number):
}
function resolveLogOrLogRange(node: SyntaxNode, text: string, pos: number, afterPipe: boolean): Situation | null {
// here the `node` is either a LogExpr or a LogRangeExpr
// we want to handle the case where we are next to a selector
// Here the `node` is either a LogExpr or a LogRangeExpr
// We want to handle the case where we are next to a selector
const selectorNode = walk(node, [['firstChild', Selector]]);
// we check that the selector is before the cursor, not after it
if (selectorNode != null && selectorNode.to <= pos) {
const labels = getLabels(selectorNode, text);
return {
type: 'AFTER_SELECTOR',
afterPipe,
labels,
};
// Check that the selector is before the cursor, not after it
if (!selectorNode || selectorNode.to > pos) {
return null;
}
return null;
return {
type: 'AFTER_SELECTOR',
afterPipe,
logQuery: getLogQueryFromMetricsQuery(text).trim(),
};
}
function resolveSelector(node: SyntaxNode, text: string, pos: number): Situation | null {

14
public/app/plugins/datasource/loki/queryUtils.test.ts
View File

@ -6,6 +6,7 @@ import {
isQueryWithParser,
isValidQuery,
parseToNodeNamesArray,
getParserFromQuery,
} from './queryUtils';
import { LokiQuery, LokiQueryType } from './types';
@ -249,3 +250,16 @@ describe('isQueryWithLabelFormat', () => {
expect(isQueryWithLabelFormat('rate({job="grafana"} [5m])')).toBe(false);
});
});
describe('getParserFromQuery', () => {
it('returns no parser', () => {
expect(getParserFromQuery('{job="grafana"}')).toBeUndefined();
});
it.each(['json', 'logfmt', 'pattern', 'regexp', 'unpack'])('detects %s parser', (parser: string) => {
expect(getParserFromQuery(`{job="grafana"} | ${parser}`)).toBe(parser);
expect(getParserFromQuery(`sum(count_over_time({place="luna"} | ${parser} | unwrap counter )) by (place)`)).toBe(
parser
);
});
});

15
public/app/plugins/datasource/loki/queryUtils.ts
View File

@ -158,6 +158,21 @@ export function isQueryWithParser(query: string): { queryWithParser: boolean; pa
return { queryWithParser: parserCount > 0, parserCount };
}
export function getParserFromQuery(query: string) {
const tree = parser.parse(query);
let logParser;
tree.iterate({
enter: (node: SyntaxNode): false | void => {
if (node.type.id === LabelParser || node.type.id === JsonExpressionParser) {
logParser = query.substring(node.from, node.to).trim();
return false;
}
},
});
return logParser;
}
export function isQueryPipelineErrorFiltering(query: string): boolean {
let isQueryPipelineErrorFiltering = false;
const tree = parser.parse(query);