Loki: Added support for "or" statements in line filters (#78705)

* Lezer: upgrade to 0.2.2

* Operations: update definitions

* Operations: update renderer

* Parsing: parse line filters with or operations

* Parsing: add unit test

* Formatting

* getHighlighterExpressionsFromQuery: add support for or statements

* Operation editor: trim button title if param name is empty

* getHighlighterExpressionsFromQuery: properly handle ip filters

package.json

@@ -250,7 +250,7 @@
     "@grafana/faro-web-sdk": "1.2.1",
     "@grafana/flamegraph": "workspace:*",
     "@grafana/google-sdk": "0.1.1",
-    "@grafana/lezer-logql": "0.2.1",
+    "@grafana/lezer-logql": "0.2.2",
     "@grafana/lezer-traceql": "0.0.11",
     "@grafana/monaco-logql": "^0.0.7",
     "@grafana/runtime": "workspace:*",

public/app/plugins/datasource/loki/backendResultTransformer.ts

@@ -2,7 +2,7 @@ import { DataQueryResponse, DataFrame, isDataFrame, FieldType, QueryResultMeta,
 
 import { getDerivedFields } from './getDerivedFields';
 import { makeTableFrames } from './makeTableFrames';
-import { formatQuery, getHighlighterExpressionsFromQuery } from './queryUtils';
+import { getHighlighterExpressionsFromQuery } from './queryUtils';
 import { dataFrameHasLokiError } from './responseUtils';
 import { DerivedFieldConfig, LokiQuery, LokiQueryType } from './types';
 
@@ -39,7 +39,7 @@ function processStreamFrame(
   const meta: QueryResultMeta = {
     preferredVisualisationType: 'logs',
     limit: query?.maxLines,
-    searchWords: query !== undefined ? getHighlighterExpressionsFromQuery(formatQuery(query.expr)) : undefined,
+    searchWords: query !== undefined ? getHighlighterExpressionsFromQuery(query.expr) : undefined,
     custom,
   };
 

public/app/plugins/datasource/loki/queryUtils.test.ts

@@ -121,6 +121,14 @@ describe('getHighlighterExpressionsFromQuery', () => {
   `('should correctly identify the type of quote used in the term', ({ input, expected }) => {
     expect(getHighlighterExpressionsFromQuery(`{foo="bar"} |= ${input}`)).toEqual([expected]);
   });
+
+  it.each(['|=', '|~'])('returns multiple expressions when using or statements', (op: string) => {
+    expect(getHighlighterExpressionsFromQuery(`{app="frontend"} ${op} "line" or "text"`)).toEqual(['line', 'text']);
+  });
+
+  it.each(['|=', '|~'])('returns multiple expressions when using or statements and ip filters', (op: string) => {
+    expect(getHighlighterExpressionsFromQuery(`{app="frontend"} ${op} "line" or ip("10.0.0.1")`)).toEqual(['line']);
+  });
 });
 
 describe('getNormalizedLokiQuery', () => {

public/app/plugins/datasource/loki/queryUtils.ts

@@ -21,6 +21,8 @@ import {
   formatLokiQuery,
   Logfmt,
   Json,
+  OrFilter,
+  FilterOp,
 } from '@grafana/lezer-logql';
 import { reportInteraction } from '@grafana/runtime';
 import { DataQuery } from '@grafana/schema';
@@ -32,55 +34,67 @@ import { LokiDatasource } from './datasource';
 import { getStreamSelectorPositions, NodePosition } from './modifyQuery';
 import { LokiQuery, LokiQueryType } from './types';
 
-export function formatQuery(selector: string | undefined): string {
-  return `${selector || ''}`.trim();
-}
-
 /**
  * Returns search terms from a LogQL query.
  * E.g., `{} |= foo |=bar != baz` returns `['foo', 'bar']`.
  */
-export function getHighlighterExpressionsFromQuery(input: string): string[] {
+export function getHighlighterExpressionsFromQuery(input = ''): string[] {
   const results = [];
 
   const filters = getNodesFromQuery(input, [LineFilter]);
 
-  for (let filter of filters) {
+  for (const filter of filters) {
     const pipeExact = filter.getChild(Filter)?.getChild(PipeExact);
     const pipeMatch = filter.getChild(Filter)?.getChild(PipeMatch);
-    const string = filter.getChild(String);
+    const strings = getStringsFromLineFilter(filter);
 
-    if ((!pipeExact && !pipeMatch) || !string) {
+    if ((!pipeExact && !pipeMatch) || !strings.length) {
       continue;
     }
 
-    const filterTerm = input.substring(string.from, string.to).trim();
-    const backtickedTerm = filterTerm[0] === '`';
-    const unwrappedFilterTerm = filterTerm.substring(1, filterTerm.length - 1);
+    for (const string of strings) {
+      const filterTerm = input.substring(string.from, string.to).trim();
+      const backtickedTerm = filterTerm[0] === '`';
+      const unwrappedFilterTerm = filterTerm.substring(1, filterTerm.length - 1);
 
-    if (!unwrappedFilterTerm) {
-      continue;
-    }
+      if (!unwrappedFilterTerm) {
+        continue;
+      }
 
-    let resultTerm = '';
+      let resultTerm = '';
 
-    // Only filter expressions with |~ operator are treated as regular expressions
-    if (pipeMatch) {
-      // When using backticks, Loki doesn't require to escape special characters and we can just push regular expression to highlights array
-      // When using quotes, we have extra backslash escaping and we need to replace \\ with \
-      resultTerm = backtickedTerm ? unwrappedFilterTerm : unwrappedFilterTerm.replace(/\\\\/g, '\\');
-    } else {
-      // We need to escape this string so it is not matched as regular expression
-      resultTerm = escapeRegExp(unwrappedFilterTerm);
-    }
+      // Only filter expressions with |~ operator are treated as regular expressions
+      if (pipeMatch) {
+        // When using backticks, Loki doesn't require to escape special characters and we can just push regular expression to highlights array
+        // When using quotes, we have extra backslash escaping and we need to replace \\ with \
+        resultTerm = backtickedTerm ? unwrappedFilterTerm : unwrappedFilterTerm.replace(/\\\\/g, '\\');
+      } else {
+        // We need to escape this string so it is not matched as regular expression
+        resultTerm = escapeRegExp(unwrappedFilterTerm);
+      }
 
-    if (resultTerm) {
-      results.push(resultTerm);
+      if (resultTerm) {
+        results.push(resultTerm);
+      }
     }
   }
   return results;
 }
 
+export function getStringsFromLineFilter(filter: SyntaxNode): SyntaxNode[] {
+  const nodes: SyntaxNode[] = [];
+  let node: SyntaxNode | null = filter;
+  do {
+    const string = node.getChild(String);
+    if (string && !node.getChild(FilterOp)) {
+      nodes.push(string);
+    }
+    node = node.getChild(OrFilter);
+  } while (node != null);
+
+  return nodes;
+}
+
 export function getNormalizedLokiQuery(query: LokiQuery): LokiQuery {
   const queryType = getLokiQueryType(query);
   // instant and range are deprecated, we want to remove them

public/app/plugins/datasource/loki/querybuilder/operationUtils.ts

@@ -296,9 +296,9 @@ export function addNestedQueryHandler(def: QueryBuilderOperationDef, query: Loki
 export function getLineFilterRenderer(operation: string, caseInsensitive?: boolean) {
   return function lineFilterRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {
     if (caseInsensitive) {
-      return `${innerExpr} ${operation} \`(?i)${model.params[0]}\``;
+      return `${innerExpr} ${operation} \`(?i)${model.params.join('` or `(?i)')}\``;
     }
-    return `${innerExpr} ${operation} \`${model.params[0]}\``;
+    return `${innerExpr} ${operation} \`${model.params.join('` or `')}\``;
   };
 }
 

public/app/plugins/datasource/loki/querybuilder/operations.ts

@@ -246,9 +246,10 @@ Example: \`\`error_level=\`level\` \`\`
       name: 'Line contains',
       params: [
         {
-          name: 'String',
+          name: '',
           type: 'string',
           hideName: true,
+          restParam: true,
           placeholder: 'Text to find',
           description: 'Find log lines that contains this text',
           minWidth: 20,
@@ -261,16 +262,17 @@ Example: \`\`error_level=\`level\` \`\`
       orderRank: LokiOperationOrder.LineFilters,
       renderer: getLineFilterRenderer('|='),
       addOperationHandler: addLokiOperation,
-      explainHandler: (op) => `Return log lines that contain string \`${op.params[0]}\`.`,
+      explainHandler: (op) => `Return log lines that contain string \`${op.params?.join('`, or `')}\`.`,
     },
     {
       id: LokiOperationId.LineContainsNot,
       name: 'Line does not contain',
       params: [
         {
-          name: 'String',
+          name: '',
           type: 'string',
           hideName: true,
+          restParam: true,
           placeholder: 'Text to exclude',
           description: 'Find log lines that does not contain this text',
           minWidth: 26,
@@ -283,16 +285,17 @@ Example: \`\`error_level=\`level\` \`\`
       orderRank: LokiOperationOrder.LineFilters,
       renderer: getLineFilterRenderer('!='),
       addOperationHandler: addLokiOperation,
-      explainHandler: (op) => `Return log lines that does not contain string \`${op.params[0]}\`.`,
+      explainHandler: (op) => `Return log lines that does not contain string \`${op.params?.join('`, or `')}\`.`,
     },
     {
       id: LokiOperationId.LineContainsCaseInsensitive,
       name: 'Line contains case insensitive',
       params: [
         {
-          name: 'String',
+          name: '',
           type: 'string',
           hideName: true,
+          restParam: true,
           placeholder: 'Text to find',
           description: 'Find log lines that contains this text',
           minWidth: 33,
@@ -305,16 +308,17 @@ Example: \`\`error_level=\`level\` \`\`
       orderRank: LokiOperationOrder.LineFilters,
       renderer: getLineFilterRenderer('|~', true),
       addOperationHandler: addLokiOperation,
-      explainHandler: (op) => `Return log lines that match regex \`(?i)${op.params[0]}\`.`,
+      explainHandler: (op) => `Return log lines that match regex \`(?i)${op.params?.join('`, or `(?i)')}\`.`,
     },
     {
       id: LokiOperationId.LineContainsNotCaseInsensitive,
       name: 'Line does not contain case insensitive',
       params: [
         {
-          name: 'String',
+          name: '',
           type: 'string',
           hideName: true,
+          restParam: true,
           placeholder: 'Text to exclude',
           description: 'Find log lines that does not contain this text',
           minWidth: 40,
@@ -327,16 +331,17 @@ Example: \`\`error_level=\`level\` \`\`
       orderRank: LokiOperationOrder.LineFilters,
       renderer: getLineFilterRenderer('!~', true),
       addOperationHandler: addLokiOperation,
-      explainHandler: (op) => `Return log lines that does not match regex \`(?i)${op.params[0]}\`.`,
+      explainHandler: (op) => `Return log lines that does not match regex \`(?i)${op.params?.join('`, or `(?i)')}\`.`,
     },
     {
       id: LokiOperationId.LineMatchesRegex,
       name: 'Line contains regex match',
       params: [
         {
-          name: 'Regex',
+          name: '',
           type: 'string',
           hideName: true,
+          restParam: true,
           placeholder: 'Pattern to match',
           description: 'Find log lines that match this regex pattern',
           minWidth: 30,
@@ -349,16 +354,17 @@ Example: \`\`error_level=\`level\` \`\`
       orderRank: LokiOperationOrder.LineFilters,
       renderer: getLineFilterRenderer('|~'),
       addOperationHandler: addLokiOperation,
-      explainHandler: (op) => `Return log lines that match a \`RE2\` regex pattern. \`${op.params[0]}\`.`,
+      explainHandler: (op) => `Return log lines that match a \`RE2\` regex pattern. \`${op.params?.join('`, or `')}\`.`,
     },
     {
       id: LokiOperationId.LineMatchesRegexNot,
       name: 'Line does not match regex',
       params: [
         {
-          name: 'Regex',
+          name: '',
           type: 'string',
           hideName: true,
+          restParam: true,
           placeholder: 'Pattern to exclude',
           description: 'Find log lines that does not match this regex pattern',
           minWidth: 30,
@@ -371,7 +377,8 @@ Example: \`\`error_level=\`level\` \`\`
       orderRank: LokiOperationOrder.LineFilters,
       renderer: getLineFilterRenderer('!~'),
       addOperationHandler: addLokiOperation,
-      explainHandler: (op) => `Return log lines that doesn't match a \`RE2\` regex pattern. \`${op.params[0]}\`.`,
+      explainHandler: (op) =>
+        `Return log lines that doesn't match a \`RE2\` regex pattern. \`${op.params?.join('`, or `')}\`.`,
     },
     {
       id: LokiOperationId.LineFilterIpMatches,

public/app/plugins/datasource/loki/querybuilder/parsing.test.ts

@@ -171,6 +171,26 @@ describe('buildVisualQueryFromString', () => {
     );
   });
 
+  it.each([
+    ['|=', LokiOperationId.LineContains],
+    ['!=', LokiOperationId.LineContainsNot],
+    ['|~', LokiOperationId.LineMatchesRegex],
+    ['!~', LokiOperationId.LineMatchesRegexNot],
+  ])('parses query with line filter and `or` statements', (op: string, id: LokiOperationId) => {
+    expect(buildVisualQueryFromString(`{app="frontend"} ${op} "line" or "text"`)).toEqual(
+      noErrors({
+        labels: [
+          {
+            op: '=',
+            value: 'frontend',
+            label: 'app',
+          },
+        ],
+        operations: [{ id, params: ['line', 'text'] }],
+      })
+    );
+  });
+
   it('parses query with line filters and escaped characters', () => {
     expect(buildVisualQueryFromString('{app="frontend"} |= "\\\\line"')).toEqual(
       noErrors({

public/app/plugins/datasource/loki/querybuilder/parsing.ts

@@ -51,6 +51,7 @@ import {
   Without,
   BinOpModifier,
   OnOrIgnoringModifier,
+  OrFilter,
 } from '@grafana/lezer-logql';
 
 import {
@@ -275,7 +276,6 @@ function getLineFilter(expr: string, node: SyntaxNode): GetOperationResult {
   const filter = getString(expr, node.getChild(Filter));
   const filterExpr = handleQuotes(getString(expr, node.getChild(String)));
   const ipLineFilter = node.getChild(FilterOp)?.getChild(Ip);
-
   if (ipLineFilter) {
     return {
       operation: {
@@ -284,6 +284,14 @@ function getLineFilter(expr: string, node: SyntaxNode): GetOperationResult {
       },
     };
   }
+
+  const params = [filterExpr];
+  let orFilter = node.getChild(OrFilter);
+  while (orFilter) {
+    params.push(handleQuotes(getString(expr, orFilter.getChild(String))));
+    orFilter = orFilter.getChild(OrFilter);
+  }
+
   const mapFilter: Record<string, LokiOperationId> = {
     '|=': LokiOperationId.LineContains,
     '!=': LokiOperationId.LineContainsNot,
@@ -294,7 +302,7 @@ function getLineFilter(expr: string, node: SyntaxNode): GetOperationResult {
   return {
     operation: {
       id: mapFilter[filter],
-      params: [filterExpr],
+      params,
     },
   };
 }

public/app/plugins/datasource/prometheus/querybuilder/shared/OperationEditor.tsx

@@ -218,7 +218,7 @@ function renderAddRestParamButton(
       <Button
         size="sm"
         icon="plus"
-        title={`Add ${paramDef.name}`}
+        title={`Add ${paramDef.name}`.trimEnd()}
         variant="secondary"
         onClick={onAddRestParam}
         data-testid={`operations.${operationIndex}.add-rest-param`}

yarn.lock

@@ -3215,12 +3215,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@grafana/lezer-logql@npm:0.2.1":
-  version: 0.2.1
-  resolution: "@grafana/lezer-logql@npm:0.2.1"
+"@grafana/lezer-logql@npm:0.2.2":
+  version: 0.2.2
+  resolution: "@grafana/lezer-logql@npm:0.2.2"
   peerDependencies:
     "@lezer/lr": ^1.0.0
-  checksum: e3669e8e1b41eb87547756fb592681aec9d728397b7bb0ccd4cdb875632fabd3469321b6565da814f5700dece28918a1770313da2364a5bc6e3745ef96d10461
+  checksum: 4a2aa48c67a75a246a13e62854470aa36f15087e212b81cdba5fa1ac913a1bdd47da374fa47576c7db670a7333af460aff95465c61ed3cd48a77df6b918d812c
   languageName: node
   linkType: hard
 
@@ -17310,7 +17310,7 @@ __metadata:
     "@grafana/faro-web-sdk": "npm:1.2.1"
     "@grafana/flamegraph": "workspace:*"
     "@grafana/google-sdk": "npm:0.1.1"
-    "@grafana/lezer-logql": "npm:0.2.1"
+    "@grafana/lezer-logql": "npm:0.2.2"
     "@grafana/lezer-traceql": "npm:0.0.11"
     "@grafana/monaco-logql": "npm:^0.0.7"
     "@grafana/runtime": "workspace:*"