From fc9b8f6be11fca800c1641e17da3ca02f7a5aa1a Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Wed, 23 Aug 2023 14:52:19 +0200 Subject: [PATCH] Loki: Implement `keep` and `drop` operations (#73636) * Update lezer * Add functionalities for code and builder * Add comment --- package.json | 2 +- .../completions.test.ts | 38 ++++++++ .../monaco-completion-provider/completions.ts | 28 ++++++ .../situation.test.ts | 32 +++++++ .../monaco-completion-provider/situation.ts | 46 ++++++++++ .../loki/querybuilder/operations.ts | 49 ++++++++++ .../loki/querybuilder/parsing.test.ts | 90 +++++++++++++++++++ .../datasource/loki/querybuilder/parsing.ts | 50 +++++++++++ .../datasource/loki/querybuilder/types.ts | 2 + yarn.lock | 10 +-- 10 files changed, 341 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 053cfabbfab..8885ead4896 100644 --- a/package.json +++ b/package.json @@ -240,7 +240,7 @@ "@grafana/faro-core": "1.1.2", "@grafana/faro-web-sdk": "1.1.2", "@grafana/google-sdk": "0.1.1", - "@grafana/lezer-logql": "0.1.8", + "@grafana/lezer-logql": "0.1.9", "@grafana/lezer-traceql": "0.0.4", "@grafana/monaco-logql": "^0.0.7", "@grafana/runtime": "workspace:*", diff --git a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.test.ts b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.test.ts index 184cfe87dc9..7e8d735d3a4 100644 --- a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.test.ts +++ b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.test.ts @@ -130,6 +130,18 @@ const afterSelectorCompletions = [ label: 'distinct', type: 'PIPE_OPERATION', }, + { + documentation: 'Operator docs', + insertText: '| drop', + label: 'drop', + type: 'PIPE_OPERATION', + }, + { + documentation: 'Operator docs', + insertText: '| keep', + label: 'keep', + type: 'PIPE_OPERATION', + }, ]; function buildAfterSelectorCompletions( @@ -413,6 +425,32 @@ describe('getCompletions', () => { }, ]); }); + + test('Returns completion options when the situation is AFTER_KEEP_AND_DROP', async () => { + const situation: Situation = { type: 'AFTER_KEEP_AND_DROP', logQuery: '{label="value"}' }; + const completions = await getCompletions(situation, completionProvider); + + expect(completions).toEqual([ + { + insertText: 'extracted', + label: 'extracted', + triggerOnInsert: false, + type: 'LABEL_NAME', + }, + { + insertText: 'place', + label: 'place', + triggerOnInsert: false, + type: 'LABEL_NAME', + }, + { + insertText: 'source', + label: 'source', + triggerOnInsert: false, + type: 'LABEL_NAME', + }, + ]); + }); }); describe('getAfterSelectorCompletions', () => { diff --git a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.ts b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.ts index 785a8965cc1..73abf56b69a 100644 --- a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.ts +++ b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.ts @@ -293,6 +293,20 @@ export async function getAfterSelectorCompletions( documentation: explainOperator(LokiOperationId.Distinct), }); + completions.push({ + type: 'PIPE_OPERATION', + label: 'drop', + insertText: `${prefix}drop`, + documentation: explainOperator(LokiOperationId.Drop), + }); + + completions.push({ + type: 'PIPE_OPERATION', + label: 'keep', + insertText: `${prefix}keep`, + documentation: explainOperator(LokiOperationId.Keep), + }); + // Let's show label options only if query has parser if (hasQueryParser) { extractedLabelKeys.forEach((key) => { @@ -357,6 +371,18 @@ async function getAfterDistinctCompletions(logQuery: string, dataProvider: Compl return [...labelCompletions]; } +async function getAfterKeepAndDropCompletions(logQuery: string, dataProvider: CompletionDataProvider) { + const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery); + const labelCompletions: Completion[] = extractedLabelKeys.map((label) => ({ + type: 'LABEL_NAME', + label, + insertText: label, + triggerOnInsert: false, + })); + + return [...labelCompletions]; +} + export async function getCompletions( situation: Situation, dataProvider: CompletionDataProvider @@ -393,6 +419,8 @@ export async function getCompletions( return [...FUNCTION_COMPLETIONS, ...AGGREGATION_COMPLETIONS]; case 'AFTER_DISTINCT': return getAfterDistinctCompletions(situation.logQuery, dataProvider); + case 'AFTER_KEEP_AND_DROP': + return getAfterKeepAndDropCompletions(situation.logQuery, dataProvider); default: throw new NeverCaseError(situation); } diff --git a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.test.ts b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.test.ts index 6077b769a9e..a12be3de07e 100644 --- a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.test.ts +++ b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.test.ts @@ -276,4 +276,36 @@ describe('situation', () => { logQuery: '{label="value"} | logfmt ', }); }); + + it('identifies AFTER_KEEP_AND_DROP autocomplete situations', () => { + assertSituation('{label="value"} | logfmt | drop^', { + type: 'AFTER_KEEP_AND_DROP', + logQuery: '{label="value"} | logfmt ', + }); + + assertSituation('{label="value"} | logfmt | keep^', { + type: 'AFTER_KEEP_AND_DROP', + logQuery: '{label="value"} | logfmt ', + }); + + assertSituation('{label="value"} | logfmt | drop id,^', { + type: 'AFTER_KEEP_AND_DROP', + logQuery: '{label="value"} | logfmt ', + }); + + assertSituation('{label="value"} | logfmt | keep id,^', { + type: 'AFTER_KEEP_AND_DROP', + logQuery: '{label="value"} | logfmt ', + }); + + assertSituation('{label="value"} | logfmt | drop id, name="test",^', { + type: 'AFTER_KEEP_AND_DROP', + logQuery: '{label="value"} | logfmt ', + }); + + assertSituation('{label="value"} | logfmt | keep id, name="test",^', { + type: 'AFTER_KEEP_AND_DROP', + logQuery: '{label="value"} | logfmt ', + }); + }); }); diff --git a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.ts b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.ts index 2d1efbc386e..037f25ad1fd 100644 --- a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.ts +++ b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.ts @@ -22,6 +22,10 @@ import { UnwrapExpr, DistinctFilter, DistinctLabel, + DropLabelsExpr, + KeepLabelsExpr, + DropLabels, + KeepLabels, } from '@grafana/lezer-logql'; import { getLogQueryFromMetricsQuery } from '../../../queryUtils'; @@ -131,6 +135,10 @@ export type Situation = | { type: 'AFTER_DISTINCT'; logQuery: string; + } + | { + type: 'AFTER_KEEP_AND_DROP'; + logQuery: string; }; type Resolver = { @@ -205,6 +213,22 @@ const RESOLVERS: Resolver[] = [ path: [ERROR_NODE_ID, DistinctLabel], fun: resolveAfterDistinct, }, + { + path: [ERROR_NODE_ID, DropLabelsExpr], + fun: resolveAfterKeepAndDrop, + }, + { + path: [ERROR_NODE_ID, DropLabels], + fun: resolveAfterKeepAndDrop, + }, + { + path: [ERROR_NODE_ID, KeepLabelsExpr], + fun: resolveAfterKeepAndDrop, + }, + { + path: [ERROR_NODE_ID, KeepLabels], + fun: resolveAfterKeepAndDrop, + }, ]; const LABEL_OP_MAP = new Map([ @@ -532,6 +556,28 @@ function resolveAfterDistinct(node: SyntaxNode, text: string, pos: number): Situ }; } +function resolveAfterKeepAndDrop(node: SyntaxNode, text: string, pos: number): Situation | null { + let logQuery = getLogQueryFromMetricsQuery(text).trim(); + let keepAndDropParent: SyntaxNode | null = null; + let parent = node.parent; + while (parent !== null) { + if (parent.type.id === PipelineStage) { + keepAndDropParent = parent; + break; + } + parent = parent.parent; + } + + if (keepAndDropParent?.type.id === PipelineStage) { + logQuery = logQuery.slice(0, keepAndDropParent.from); + } + + return { + type: 'AFTER_KEEP_AND_DROP', + logQuery, + }; +} + // we find the first error-node in the tree that is at the cursor-position. // NOTE: this might be too slow, might need to optimize it // (ideas: we do not need to go into every subtree, based on from/to) diff --git a/public/app/plugins/datasource/loki/querybuilder/operations.ts b/public/app/plugins/datasource/loki/querybuilder/operations.ts index 4cff3263c96..0c9369c5126 100644 --- a/public/app/plugins/datasource/loki/querybuilder/operations.ts +++ b/public/app/plugins/datasource/loki/querybuilder/operations.ts @@ -508,6 +508,55 @@ Example: \`\`error_level=\`level\` \`\` explainHandler: () => 'Allows filtering log lines using their original and extracted labels to filter out duplicate label values. The first line occurrence of a distinct value is returned, and the others are dropped.', }, + { + id: LokiOperationId.Drop, + name: 'Drop', + params: [ + // As drop can support both labels (e.g. job) and expressions (e.g. job="grafana"), we + // use input and not LabelParamEditor. + { + name: 'Label', + type: 'string', + restParam: true, + optional: true, + minWidth: 18, + placeholder: 'job="grafana"', + description: 'Specify labels or expressions to drop.', + }, + ], + defaultParams: [''], + alternativesKey: 'format', + category: LokiVisualQueryOperationCategory.Formats, + orderRank: LokiOperationOrder.PipeOperations, + renderer: (op, def, innerExpr) => `${innerExpr} | drop ${op.params.join(',')}`, + addOperationHandler: addLokiOperation, + explainHandler: () => 'The drop expression will drop the given labels in the pipeline.', + }, + { + id: LokiOperationId.Keep, + name: 'Keep', + params: [ + // As keep can support both labels (e.g. job) and expressions (e.g. job="grafana"), we + // use input and not LabelParamEditor. + { + name: 'Label', + type: 'string', + restParam: true, + optional: true, + minWidth: 18, + placeholder: 'job="grafana"', + description: 'Specify labels or expressions to keep.', + }, + ], + defaultParams: [''], + alternativesKey: 'format', + category: LokiVisualQueryOperationCategory.Formats, + orderRank: LokiOperationOrder.PipeOperations, + renderer: (op, def, innerExpr) => `${innerExpr} | keep ${op.params.join(',')}`, + addOperationHandler: addLokiOperation, + explainHandler: () => + 'The keep expression will keep only the specified labels in the pipeline and drop all the other labels.', + }, ...binaryScalarOperations, { id: LokiOperationId.NestedQuery, diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts index f9508866d47..0dcb06c6ed3 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.test.ts @@ -776,6 +776,96 @@ describe('buildVisualQueryFromString', () => { }) ); }); + + it('parses a log query with drop and no labels', () => { + expect(buildVisualQueryFromString('{app="frontend"} | drop')).toEqual( + noErrors({ + labels: [ + { + op: '=', + value: 'frontend', + label: 'app', + }, + ], + operations: [{ id: LokiOperationId.Drop, params: [] }], + }) + ); + }); + + it('parses a log query with drop and labels', () => { + expect(buildVisualQueryFromString('{app="frontend"} | drop id, email')).toEqual( + noErrors({ + labels: [ + { + op: '=', + value: 'frontend', + label: 'app', + }, + ], + operations: [{ id: LokiOperationId.Drop, params: ['id', 'email'] }], + }) + ); + }); + + it('parses a log query with drop, labels and expressions', () => { + expect(buildVisualQueryFromString('{app="frontend"} | drop id, email, test="test1"')).toEqual( + noErrors({ + labels: [ + { + op: '=', + value: 'frontend', + label: 'app', + }, + ], + operations: [{ id: LokiOperationId.Drop, params: ['id', 'email', 'test="test1"'] }], + }) + ); + }); + + it('parses a log query with keep and no labels', () => { + expect(buildVisualQueryFromString('{app="frontend"} | keep')).toEqual( + noErrors({ + labels: [ + { + op: '=', + value: 'frontend', + label: 'app', + }, + ], + operations: [{ id: LokiOperationId.Keep, params: [] }], + }) + ); + }); + + it('parses a log query with keep and labels', () => { + expect(buildVisualQueryFromString('{app="frontend"} | keep id, email')).toEqual( + noErrors({ + labels: [ + { + op: '=', + value: 'frontend', + label: 'app', + }, + ], + operations: [{ id: LokiOperationId.Keep, params: ['id', 'email'] }], + }) + ); + }); + + it('parses a log query with keep, labels and expressions', () => { + expect(buildVisualQueryFromString('{app="frontend"} | keep id, email, test="test1"')).toEqual( + noErrors({ + labels: [ + { + op: '=', + value: 'frontend', + label: 'app', + }, + ], + operations: [{ id: LokiOperationId.Keep, params: ['id', 'email', 'test="test1"'] }], + }) + ); + }); }); function noErrors(query: LokiVisualQuery) { diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.ts index 2bedcf3c227..508b7299851 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.ts @@ -10,6 +10,9 @@ import { Decolorize, DistinctFilter, DistinctLabel, + DropLabel, + DropLabels, + DropLabelsExpr, Filter, FilterOp, Grouping, @@ -21,6 +24,9 @@ import { Json, JsonExpression, JsonExpressionParser, + KeepLabel, + KeepLabels, + KeepLabelsExpr, LabelFilter, LabelFormatMatcher, LabelParser, @@ -212,6 +218,16 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex break; } + case DropLabelsExpr: { + visQuery.operations.push(handleDropFilter(expr, node, context)); + break; + } + + case KeepLabelsExpr: { + visQuery.operations.push(handleKeepFilter(expr, node, context)); + break; + } + default: { // Any other nodes we just ignore and go to its children. This should be fine as there are lots of wrapper // nodes that can be skipped. @@ -660,3 +676,37 @@ function handleDistinctFilter(expr: string, node: SyntaxNode, context: Context): params: labels, }; } + +function handleDropFilter(expr: string, node: SyntaxNode, context: Context): QueryBuilderOperation { + const labels: string[] = []; + let exploringNode = node.getChild(DropLabels); + while (exploringNode) { + const label = getString(expr, exploringNode.getChild(DropLabel)); + if (label) { + labels.push(label); + } + exploringNode = exploringNode?.getChild(DropLabels); + } + labels.reverse(); + return { + id: LokiOperationId.Drop, + params: labels, + }; +} + +function handleKeepFilter(expr: string, node: SyntaxNode, context: Context): QueryBuilderOperation { + const labels: string[] = []; + let exploringNode = node.getChild(KeepLabels); + while (exploringNode) { + const label = getString(expr, exploringNode.getChild(KeepLabel)); + if (label) { + labels.push(label); + } + exploringNode = exploringNode?.getChild(KeepLabels); + } + labels.reverse(); + return { + id: LokiOperationId.Keep, + params: labels, + }; +} diff --git a/public/app/plugins/datasource/loki/querybuilder/types.ts b/public/app/plugins/datasource/loki/querybuilder/types.ts index 6e534b62418..28350e45013 100644 --- a/public/app/plugins/datasource/loki/querybuilder/types.ts +++ b/public/app/plugins/datasource/loki/querybuilder/types.ts @@ -42,6 +42,8 @@ export enum LokiOperationId { LineFormat = 'line_format', LabelFormat = 'label_format', Decolorize = 'decolorize', + Drop = 'drop', + Keep = 'keep', Rate = 'rate', RateCounter = 'rate_counter', CountOverTime = 'count_over_time', diff --git a/yarn.lock b/yarn.lock index 7aaee77f83b..14dbad72cd3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3857,14 +3857,14 @@ __metadata: languageName: node linkType: hard -"@grafana/lezer-logql@npm:0.1.8": - version: 0.1.8 - resolution: "@grafana/lezer-logql@npm:0.1.8" +"@grafana/lezer-logql@npm:0.1.9": + version: 0.1.9 + resolution: "@grafana/lezer-logql@npm:0.1.9" dependencies: lodash: ^4.17.21 peerDependencies: "@lezer/lr": ^1.0.0 - checksum: f0f301b6d4fbd2d79563b5b4e34303257be0ea995b2b9fa1f012648654b4afaa9cea91642bc59eddb70e9fa24ec8804489c161f7065b41eef49db68d3a2ca561 + checksum: 6cceb18586413864137ef2305bbe2fdce054796c61a3fde4864e3f2d30ea3dac853aa701728867bf609a4009334dbf92fddbc06f7291dbca9459a86c9dc29e67 languageName: node linkType: hard @@ -19244,7 +19244,7 @@ __metadata: "@grafana/faro-core": 1.1.2 "@grafana/faro-web-sdk": 1.1.2 "@grafana/google-sdk": 0.1.1 - "@grafana/lezer-logql": 0.1.8 + "@grafana/lezer-logql": 0.1.9 "@grafana/lezer-traceql": 0.0.4 "@grafana/monaco-logql": ^0.0.7 "@grafana/runtime": "workspace:*"