From f8a1e7d50012c3f00c0d9b10c24a20fe59b4ced2 Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Fri, 15 Dec 2023 17:15:20 +0100 Subject: [PATCH] Loki: Decouple from Prometheus parsingUtils (#79460) Loki: Remove dependency on parsingUtils --- .betterer.results | 3 - .../monaco-completion-provider/validation.ts | 2 +- .../app/plugins/datasource/loki/datasource.ts | 2 +- .../app/plugins/datasource/loki/queryUtils.ts | 3 +- .../datasource/loki/querybuilder/parsing.ts | 18 +-- .../loki/querybuilder/parsingUtils.test.ts | 42 ++++++ .../querybuilder}/parsingUtils.ts | 74 +-------- .../querybuilder/components/MetricSelect.tsx | 2 +- .../components/MetricsLabelsSection.tsx | 2 +- .../components/metrics-modal/state/helpers.ts | 2 +- .../prometheus/querybuilder/parsing.ts | 2 +- .../{shared => }/parsingUtils.test.ts | 0 .../prometheus/querybuilder/parsingUtils.ts | 140 ++++++++++++++++++ 13 files changed, 200 insertions(+), 92 deletions(-) create mode 100644 public/app/plugins/datasource/loki/querybuilder/parsingUtils.test.ts rename public/app/plugins/datasource/{prometheus/querybuilder/shared => loki/querybuilder}/parsingUtils.ts (73%) rename public/app/plugins/datasource/prometheus/querybuilder/{shared => }/parsingUtils.test.ts (100%) create mode 100644 public/app/plugins/datasource/prometheus/querybuilder/parsingUtils.ts diff --git a/.betterer.results b/.betterer.results index fb06f6296fa..65831dbd4cc 100644 --- a/.betterer.results +++ b/.betterer.results @@ -5884,9 +5884,6 @@ exports[`better eslint`] = { [0, 0, 0, "Styles should be written using objects.", "0"], [0, 0, 0, "Styles should be written using objects.", "1"] ], - "public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts:5381": [ - [0, 0, 0, "Unexpected any. Specify a different type.", "0"] - ], "public/app/plugins/datasource/prometheus/querybuilder/shared/types.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"], diff --git a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/validation.ts b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/validation.ts index 59b4572a767..73e5e799d55 100644 --- a/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/validation.ts +++ b/public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/validation.ts @@ -1,7 +1,7 @@ import { SyntaxNode } from '@lezer/common'; import { LRParser } from '@lezer/lr'; -import { ErrorId } from 'app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils'; +import { ErrorId } from '../../../querybuilder/parsingUtils'; interface ParserErrorBoundary { startLineNumber: number; diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index f72d7b82dfa..bb65daedf98 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -48,7 +48,6 @@ import { DataQuery } from '@grafana/schema'; import { queryLogsSample, queryLogsVolume } from '../../../features/logs/logsModel'; import { getLogLevelFromKey } from '../../../features/logs/utils'; -import { replaceVariables, returnVariables } from '../prometheus/querybuilder/shared/parsingUtils'; import LanguageProvider from './LanguageProvider'; import { LiveStreams, LokiLiveTarget } from './LiveStreams'; @@ -86,6 +85,7 @@ import { isQueryWithError, requestSupportsSplitting, } from './queryUtils'; +import { replaceVariables, returnVariables } from './querybuilder/parsingUtils'; import { convertToWebSocketUrl, doLokiChannelStream } from './streaming'; import { trackQuery } from './tracking'; import { diff --git a/public/app/plugins/datasource/loki/queryUtils.ts b/public/app/plugins/datasource/loki/queryUtils.ts index de0870d923a..02a070ce4ec 100644 --- a/public/app/plugins/datasource/loki/queryUtils.ts +++ b/public/app/plugins/datasource/loki/queryUtils.ts @@ -27,11 +27,10 @@ import { import { reportInteraction } from '@grafana/runtime'; import { DataQuery } from '@grafana/schema'; -import { ErrorId, replaceVariables, returnVariables } from '../prometheus/querybuilder/shared/parsingUtils'; - import { placeHolderScopedVars } from './components/monaco-query-field/monaco-completion-provider/validation'; import { LokiDatasource } from './datasource'; import { getStreamSelectorPositions, NodePosition } from './modifyQuery'; +import { ErrorId, replaceVariables, returnVariables } from './querybuilder/parsingUtils'; import { LokiQuery, LokiQueryType } from './types'; /** diff --git a/public/app/plugins/datasource/loki/querybuilder/parsing.ts b/public/app/plugins/datasource/loki/querybuilder/parsing.ts index 24931d84a68..5c0375c0697 100644 --- a/public/app/plugins/datasource/loki/querybuilder/parsing.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsing.ts @@ -54,15 +54,6 @@ import { OrFilter, } from '@grafana/lezer-logql'; -import { - ErrorId, - getAllByType, - getLeftMostChild, - getString, - makeBinOp, - makeError, - replaceVariables, -} from '../../prometheus/querybuilder/shared/parsingUtils'; import { QueryBuilderLabelFilter, QueryBuilderOperation, @@ -71,6 +62,15 @@ import { import { binaryScalarDefs } from './binaryScalarOperations'; import { checkParamsAreValid, getDefinitionById } from './operations'; +import { + ErrorId, + getAllByType, + getLeftMostChild, + getString, + makeBinOp, + makeError, + replaceVariables, +} from './parsingUtils'; import { LokiOperationId, LokiVisualQuery, LokiVisualQueryBinary } from './types'; interface Context { diff --git a/public/app/plugins/datasource/loki/querybuilder/parsingUtils.test.ts b/public/app/plugins/datasource/loki/querybuilder/parsingUtils.test.ts new file mode 100644 index 00000000000..df691789403 --- /dev/null +++ b/public/app/plugins/datasource/loki/querybuilder/parsingUtils.test.ts @@ -0,0 +1,42 @@ +import { parser } from '@grafana/lezer-logql'; + +import { getLeftMostChild, getString, replaceVariables } from './parsingUtils'; + +describe('getLeftMostChild', () => { + it('return left most child', () => { + const tree = parser.parse('count_over_time({bar="baz"}[5m])'); + const child = getLeftMostChild(tree.topNode); + expect(child).toBeDefined(); + expect(child!.name).toBe('CountOverTime'); + }); +}); + +describe('replaceVariables', () => { + it('should replace variables', () => { + expect(replaceVariables('rate([{bar="${app}", baz="[[label_var]]"}[$__auto])')).toBe( + 'rate([{bar="__V_2__app__V__", baz="__V_1__label_var__V__"}[__V_0____auto__V__])' + ); + }); +}); + +describe('getString', () => { + it('should return correct string representation of the node', () => { + const expr = 'count_over_time({bar="baz"}[5m])'; + const tree = parser.parse(expr); + const child = getLeftMostChild(tree.topNode); + expect(getString(expr, child)).toBe('count_over_time'); + }); + + it('should return string with correct variables', () => { + const expr = 'count_over_time({bar="__V_2__app__V__"}[__V_0____auto__V__])'; + const tree = parser.parse(expr); + expect(getString(expr, tree.topNode)).toBe('count_over_time({bar="${app}"}[$__auto])'); + }); + + it('is symmetrical with replaceVariables', () => { + const expr = 'count_over_time({bar="${app}", baz="[[label_var]]"}[$__auto])'; + const replaced = replaceVariables(expr); + const tree = parser.parse(replaced); + expect(getString(replaced, tree.topNode)).toBe(expr); + }); +}); diff --git a/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts b/public/app/plugins/datasource/loki/querybuilder/parsingUtils.ts similarity index 73% rename from public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts rename to public/app/plugins/datasource/loki/querybuilder/parsingUtils.ts index d0306411e2f..a4e7215c597 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.ts +++ b/public/app/plugins/datasource/loki/querybuilder/parsingUtils.ts @@ -1,8 +1,8 @@ import { SyntaxNode, TreeCursor } from '@lezer/common'; -import { QueryBuilderOperation, QueryBuilderOperationParamValue } from './types'; +import { QueryBuilderOperation, QueryBuilderOperationParamValue } from '../../prometheus/querybuilder/shared/types'; -// Although 0 isn't explicitly provided in the lezer-promql & @grafana/lezer-logql library as the error node ID, it does appear to be the ID of error nodes within lezer. +// Although 0 isn't explicitly provided in the @grafana/lezer-logql library as the error node ID, it does appear to be the ID of error nodes within lezer. export const ErrorId = 0; export function getLeftMostChild(cur: SyntaxNode): SyntaxNode { @@ -131,76 +131,6 @@ export function getAllByType(expr: string, cur: SyntaxNode, type: number | strin return values; } -// Debugging function for convenience. Gives you nice output similar to linux tree util. -// @ts-ignore -export function log(expr: string, cur?: SyntaxNode) { - if (!cur) { - console.log(''); - return; - } - const json = toJson(expr, cur); - const text = jsonToText(json); - - if (!text) { - console.log(''); - return; - } - console.log(text); -} - -function toJson(expr: string, cur: SyntaxNode) { - const treeJson: any = {}; - const name = nodeToString(expr, cur); - const children = []; - - let pos = 0; - let child = cur.childAfter(pos); - while (child) { - children.push(toJson(expr, child)); - pos = child.to; - child = cur.childAfter(pos); - } - - treeJson.name = name; - treeJson.children = children; - return treeJson; -} - -type JsonNode = { - name: string; - children: JsonNode[]; -}; - -function jsonToText( - node: JsonNode, - context: { lastChild: boolean; indent: string } = { - lastChild: true, - indent: '', - } -) { - const name = node.name; - const { lastChild, indent } = context; - const newIndent = indent !== '' ? indent + (lastChild ? '└─' : '├─') : ''; - let text = newIndent + name; - - const children = node.children; - children.forEach((child, index) => { - const isLastChild = index === children.length - 1; - text += - '\n' + - jsonToText(child, { - lastChild: isLastChild, - indent: indent + (lastChild ? ' ' : '│ '), - }); - }); - - return text; -} - -function nodeToString(expr: string, node: SyntaxNode) { - return node.name + ': ' + getString(expr, node); -} - /** * There aren't any spaces in the metric names, so let's introduce a wildcard into the regex for each space to better facilitate a fuzzy search */ diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx index e97de0c3267..8e9067f71f8 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/MetricSelect.tsx @@ -22,7 +22,7 @@ import { SelectMenuOptions } from '@grafana/ui/src/components/Select/SelectMenu' import { PrometheusDatasource } from '../../datasource'; import { truncateResult } from '../../language_utils'; -import { regexifyLabelValuesQueryString } from '../shared/parsingUtils'; +import { regexifyLabelValuesQueryString } from '../parsingUtils'; import { QueryBuilderLabelFilter } from '../shared/types'; import { PromVisualQuery } from '../types'; diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/MetricsLabelsSection.tsx b/public/app/plugins/datasource/prometheus/querybuilder/components/MetricsLabelsSection.tsx index f6b4f26f47e..49d2bcd821f 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/MetricsLabelsSection.tsx +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/MetricsLabelsSection.tsx @@ -6,7 +6,7 @@ import { PrometheusDatasource } from '../../datasource'; import { getMetadataString } from '../../language_provider'; import { truncateResult } from '../../language_utils'; import { promQueryModeller } from '../PromQueryModeller'; -import { regexifyLabelValuesQueryString } from '../shared/parsingUtils'; +import { regexifyLabelValuesQueryString } from '../parsingUtils'; import { QueryBuilderLabelFilter } from '../shared/types'; import { PromVisualQuery } from '../types'; diff --git a/public/app/plugins/datasource/prometheus/querybuilder/components/metrics-modal/state/helpers.ts b/public/app/plugins/datasource/prometheus/querybuilder/components/metrics-modal/state/helpers.ts index d5da08e3b1c..ce35ab9d62f 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/components/metrics-modal/state/helpers.ts +++ b/public/app/plugins/datasource/prometheus/querybuilder/components/metrics-modal/state/helpers.ts @@ -4,7 +4,7 @@ import { reportInteraction } from '@grafana/runtime'; import { PrometheusDatasource } from 'app/plugins/datasource/prometheus/datasource'; import { getMetadataHelp, getMetadataType } from 'app/plugins/datasource/prometheus/language_provider'; -import { regexifyLabelValuesQueryString } from '../../../shared/parsingUtils'; +import { regexifyLabelValuesQueryString } from '../../../parsingUtils'; import { QueryBuilderLabelFilter } from '../../../shared/types'; import { PromVisualQuery } from '../../../types'; import { HaystackDictionary, MetricData, MetricsData, PromFilterOption } from '../types'; diff --git a/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts b/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts index 79792b8afe2..de1486efc2a 100644 --- a/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts +++ b/public/app/plugins/datasource/prometheus/querybuilder/parsing.ts @@ -36,7 +36,7 @@ import { makeBinOp, makeError, replaceVariables, -} from './shared/parsingUtils'; +} from './parsingUtils'; import { QueryBuilderLabelFilter, QueryBuilderOperation } from './shared/types'; import { PromVisualQuery, PromVisualQueryBinary } from './types'; diff --git a/public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.test.ts b/public/app/plugins/datasource/prometheus/querybuilder/parsingUtils.test.ts similarity index 100% rename from public/app/plugins/datasource/prometheus/querybuilder/shared/parsingUtils.test.ts rename to public/app/plugins/datasource/prometheus/querybuilder/parsingUtils.test.ts diff --git a/public/app/plugins/datasource/prometheus/querybuilder/parsingUtils.ts b/public/app/plugins/datasource/prometheus/querybuilder/parsingUtils.ts new file mode 100644 index 00000000000..10be8692ebc --- /dev/null +++ b/public/app/plugins/datasource/prometheus/querybuilder/parsingUtils.ts @@ -0,0 +1,140 @@ +import { SyntaxNode, TreeCursor } from '@lezer/common'; + +import { QueryBuilderOperation, QueryBuilderOperationParamValue } from './shared/types'; + +// Although 0 isn't explicitly provided in the lezer-promql library as the error node ID, it does appear to be the ID of error nodes within lezer. +export const ErrorId = 0; + +export function getLeftMostChild(cur: SyntaxNode): SyntaxNode { + return cur.firstChild ? getLeftMostChild(cur.firstChild) : cur; +} + +export function makeError(expr: string, node: SyntaxNode) { + return { + text: getString(expr, node), + // TODO: this are positions in the string with the replaced variables. Means it cannot be used to show exact + // placement of the error for the user. We need some translation table to positions before the variable + // replace. + from: node.from, + to: node.to, + parentType: node.parent?.name, + }; +} + +// Taken from template_srv, but copied so to not mess with the regex.index which is manipulated in the service +/* + * This regex matches 3 types of variable reference with an optional format specifier + * \$(\w+) $var1 + * \[\[([\s\S]+?)(?::(\w+))?\]\] [[var2]] or [[var2:fmt2]] + * \${(\w+)(?::(\w+))?} ${var3} or ${var3:fmt3} + */ +const variableRegex = /\$(\w+)|\[\[([\s\S]+?)(?::(\w+))?\]\]|\${(\w+)(?:\.([^:^\}]+))?(?::([^\}]+))?}/g; + +/** + * As variables with $ are creating parsing errors, we first replace them with magic string that is parsable and at + * the same time we can get the variable and its format back from it. + * @param expr + */ +export function replaceVariables(expr: string) { + return expr.replace(variableRegex, (match, var1, var2, fmt2, var3, fieldPath, fmt3) => { + const fmt = fmt2 || fmt3; + let variable = var1; + let varType = '0'; + + if (var2) { + variable = var2; + varType = '1'; + } + + if (var3) { + variable = var3; + varType = '2'; + } + + return `__V_${varType}__` + variable + '__V__' + (fmt ? '__F__' + fmt + '__F__' : ''); + }); +} + +const varTypeFunc = [ + (v: string, f?: string) => `\$${v}`, + (v: string, f?: string) => `[[${v}${f ? `:${f}` : ''}]]`, + (v: string, f?: string) => `\$\{${v}${f ? `:${f}` : ''}\}`, +]; + +/** + * Get back the text with variables in their original format. + * @param expr + */ +export function returnVariables(expr: string) { + return expr.replace(/__V_(\d)__(.+?)__V__(?:__F__(\w+)__F__)?/g, (match, type, v, f) => { + return varTypeFunc[parseInt(type, 10)](v, f); + }); +} + +/** + * Get the actual string of the expression. That is not stored in the tree so we have to get the indexes from the node + * and then based on that get it from the expression. + * @param expr + * @param node + */ +export function getString(expr: string, node: SyntaxNode | TreeCursor | null | undefined) { + if (!node) { + return ''; + } + return returnVariables(expr.substring(node.from, node.to)); +} + +/** + * Create simple scalar binary op object. + * @param opDef - definition of the op to be created + * @param expr + * @param numberNode - the node for the scalar + * @param hasBool - whether operation has a bool modifier. Is used only for ops for which it makes sense. + */ +export function makeBinOp( + opDef: { id: string; comparison?: boolean }, + expr: string, + numberNode: SyntaxNode, + hasBool: boolean +): QueryBuilderOperation { + const params: QueryBuilderOperationParamValue[] = [parseFloat(getString(expr, numberNode))]; + if (opDef.comparison) { + params.push(hasBool); + } + return { + id: opDef.id, + params, + }; +} + +/** + * Get all nodes with type in the tree. This traverses the tree so it is safe only when you know there shouldn't be + * too much nesting but you just want to skip some of the wrappers. For example getting function args this way would + * not be safe is it would also find arguments of nested functions. + * @param expr + * @param cur + * @param type - can be string or number, some data-sources (loki) haven't migrated over to using numeric constants defined in the lezer parsing library (e.g. lezer-promql). + * @todo Remove string type definition when all data-sources have migrated to numeric constants + */ +export function getAllByType(expr: string, cur: SyntaxNode, type: number | string): string[] { + if (cur.type.id === type || cur.name === type) { + return [getString(expr, cur)]; + } + const values: string[] = []; + let pos = 0; + let child = cur.childAfter(pos); + while (child) { + values.push(...getAllByType(expr, child, type)); + pos = child.to; + child = cur.childAfter(pos); + } + return values; +} + +/** + * There aren't any spaces in the metric names, so let's introduce a wildcard into the regex for each space to better facilitate a fuzzy search + */ +export const regexifyLabelValuesQueryString = (query: string) => { + const queryArray = query.split(' '); + return queryArray.map((query) => `${query}.*`).join(''); +};