(feat/explore): Support for new LogQL filtering syntax (#16674)

* (feat/explore): Support for new LogQL filtering syntax

Loki is adding syntax to support chained filtering.
This PR adapts Grafana to support this.

- Send only `query` parameter in loki request
- Automatically wrap search text in simple syntax, e.g., `{} foo` is
sent as `{} |~ "foo"`.


* Adapted to regexp parameter staying on in Loki

* Dont wrap single regexp in new filter syntax

* Fix datasource test

* Fallback regexp parameter for legacy queries

* Fix search highlighting

* Make highlighting work for filter chains

* Fix datasource test

packages/grafana-ui/src/types/data.ts

@@ -21,7 +21,7 @@ export interface QueryResultMeta {
   requestId?: string;
 
   // Used in Explore for highlighting
-  search?: string;
+  searchWords?: string[];
 
   // Used in Explore to show limit applied to search result
   limit?: number;

packages/grafana-ui/src/types/datasource.ts

@@ -194,7 +194,7 @@ export abstract class ExploreDataSourceApi<
   TOptions extends DataSourceJsonData = DataSourceJsonData
 > extends DataSourceApi<TQuery, TOptions> {
   modifyQuery?(query: TQuery, action: QueryFixAction): TQuery;
-  getHighlighterExpression?(query: TQuery): string;
+  getHighlighterExpression?(query: TQuery): string[];
   languageProvider?: any;
 }
 

public/app/core/logs_model.ts

@@ -446,7 +446,7 @@ export function processLogSeriesRow(
   const timeLocal = time.format('YYYY-MM-DD HH:mm:ss');
   const logLevel = getLogLevel(message);
   const hasAnsi = hasAnsiCodes(message);
-  const search = series.meta && series.meta.search ? series.meta.search : '';
+  const searchWords = series.meta && series.meta.searchWords ? series.meta.searchWords : [];
 
   return {
     logLevel,
@@ -455,10 +455,10 @@ export function processLogSeriesRow(
     timeLocal,
     uniqueLabels,
     hasAnsi,
+    searchWords,
     entry: hasAnsi ? ansicolor.strip(message) : message,
     raw: message,
     labels: series.labels,
-    searchWords: search ? [search] : [],
     timestamp: ts,
   };
 }

public/app/core/utils/text.ts

@@ -6,7 +6,7 @@ import xss from 'xss';
  * See https://github.com/bvaughn/react-highlight-words#props
  */
 export function findHighlightChunksInText({ searchWords, textToHighlight }) {
-  return findMatchesInText(textToHighlight, searchWords.join(' '));
+  return searchWords.reduce((acc, term) => [...acc, ...findMatchesInText(textToHighlight, term)], []);
 }
 
 const cleanNeedle = (needle: string): string => {

public/app/features/explore/LogRow.tsx

@@ -133,7 +133,7 @@ export class LogRow extends PureComponent<Props, State> {
     const { entry, hasAnsi, raw } = row;
     const previewHighlights = highlighterExpressions && !_.isEqual(highlighterExpressions, row.searchWords);
     const highlights = previewHighlights ? highlighterExpressions : row.searchWords;
-    const needsHighlighter = highlights && highlights.length > 0 && highlights[0].length > 0;
+    const needsHighlighter = highlights && highlights.length > 0 && highlights[0] && highlights[0].length > 0;
     const highlightClassName = classnames('logs-row__match-highlight', {
       'logs-row__match-highlight--preview': previewHighlights,
     });

public/app/features/explore/QueryRow.tsx

@@ -98,7 +98,7 @@ export class QueryRow extends PureComponent<QueryRowProps> {
     const { datasourceInstance } = this.props;
     if (datasourceInstance.getHighlighterExpression) {
       const { exploreId } = this.props;
-      const expressions = [datasourceInstance.getHighlighterExpression(value)];
+      const expressions = datasourceInstance.getHighlighterExpression(value);
       this.props.highlightLogsExpressionAction({ exploreId, expressions });
     }
   }, 500);

public/app/plugins/datasource/loki/datasource.test.ts

@@ -61,7 +61,7 @@ describe('LokiDatasource', () => {
       backendSrvMock.datasourceRequest = jest.fn(() => Promise.resolve(testResp));
 
       const options = getQueryOptions<LokiQuery>({
-        targets: [{ expr: 'foo', refId: 'B' }],
+        targets: [{ expr: '{} foo', refId: 'B' }],
       });
 
       const res = await ds.query(options);
@@ -69,7 +69,7 @@ describe('LokiDatasource', () => {
       const seriesData = res.data[0] as SeriesData;
       expect(seriesData.rows[0][1]).toBe('hello');
       expect(seriesData.meta.limit).toBe(20);
-      expect(seriesData.meta.search).toBe('(?i)foo');
+      expect(seriesData.meta.searchWords).toEqual(['(?i)foo']);
       done();
     });
   });

public/app/plugins/datasource/loki/datasource.ts

@@ -6,7 +6,7 @@ import * as dateMath from '@grafana/ui/src/utils/datemath';
 import { addLabelToSelector } from 'app/plugins/datasource/prometheus/add_label_to_query';
 import LanguageProvider from './language_provider';
 import { logStreamToSeriesData } from './result_transformer';
-import { formatQuery, parseQuery } from './query_utils';
+import { formatQuery, parseQuery, getHighlighterExpressionsFromQuery } from './query_utils';
 
 // Types
 import {
@@ -69,12 +69,14 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
 
   prepareQueryTarget(target: LokiQuery, options: DataQueryRequest<LokiQuery>) {
     const interpolated = this.templateSrv.replace(target.expr);
+    const { query, regexp } = parseQuery(interpolated);
     const start = this.getTime(options.range.from, false);
     const end = this.getTime(options.range.to, true);
     const refId = target.refId;
     return {
       ...DEFAULT_QUERY_PARAMS,
-      ...parseQuery(interpolated),
+      query,
+      regexp,
       start,
       end,
       limit: this.maxLines,
@@ -126,14 +128,15 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
 
       for (let i = 0; i < results.length; i++) {
         const result = results[i];
-
         if (result.data) {
           const refId = queryTargets[i].refId;
           for (const stream of result.data.streams || []) {
             const seriesData = logStreamToSeriesData(stream);
             seriesData.refId = refId;
             seriesData.meta = {
-              search: queryTargets[i].regexp,
+              searchWords: getHighlighterExpressionsFromQuery(
+                formatQuery(queryTargets[i].query, queryTargets[i].regexp)
+              ),
               limit: this.maxLines,
             };
             series.push(seriesData);
@@ -160,7 +163,7 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
 
   modifyQuery(query: LokiQuery, action: any): LokiQuery {
     const parsed = parseQuery(query.expr || '');
-    let selector = parsed.query;
+    let { query: selector } = parsed;
     switch (action.type) {
       case 'ADD_FILTER': {
         selector = addLabelToSelector(selector, action.key, action.value);
@@ -173,8 +176,8 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
     return { ...query, expr: expression };
   }
 
-  getHighlighterExpression(query: LokiQuery): string {
+  getHighlighterExpression(query: LokiQuery): string[] {
-    return parseQuery(query.expr).regexp;
+    return getHighlighterExpressionsFromQuery(query.expr);
   }
 
   getTime(date, roundUp) {

public/app/plugins/datasource/loki/query_utils.test.ts

@@ -1,56 +1,87 @@
-import { parseQuery } from './query_utils';
+import { parseQuery, getHighlighterExpressionsFromQuery } from './query_utils';
+import { LokiExpression } from './types';
 
 describe('parseQuery', () => {
   it('returns empty for empty string', () => {
     expect(parseQuery('')).toEqual({
       query: '',
       regexp: '',
-    });
+    } as LokiExpression);
   });
 
   it('returns regexp for strings without query', () => {
     expect(parseQuery('test')).toEqual({
-      query: '',
+      query: 'test',
-      regexp: '(?i)test',
+      regexp: '',
-    });
+    } as LokiExpression);
   });
 
   it('returns query for strings without regexp', () => {
     expect(parseQuery('{foo="bar"}')).toEqual({
       query: '{foo="bar"}',
       regexp: '',
-    });
+    } as LokiExpression);
   });
 
   it('returns query for strings with query and search string', () => {
     expect(parseQuery('x {foo="bar"}')).toEqual({
       query: '{foo="bar"}',
       regexp: '(?i)x',
-    });
+    } as LokiExpression);
   });
 
   it('returns query for strings with query and regexp', () => {
     expect(parseQuery('{foo="bar"} x|y')).toEqual({
       query: '{foo="bar"}',
       regexp: '(?i)x|y',
-    });
+    } as LokiExpression);
   });
 
   it('returns query for selector with two labels', () => {
     expect(parseQuery('{foo="bar", baz="42"}')).toEqual({
       query: '{foo="bar", baz="42"}',
       regexp: '',
-    });
+    } as LokiExpression);
   });
 
   it('returns query and regexp with quantifiers', () => {
     expect(parseQuery('{foo="bar"} \\.java:[0-9]{1,5}')).toEqual({
       query: '{foo="bar"}',
       regexp: '(?i)\\.java:[0-9]{1,5}',
-    });
+    } as LokiExpression);
     expect(parseQuery('\\.java:[0-9]{1,5} {foo="bar"}')).toEqual({
       query: '{foo="bar"}',
       regexp: '(?i)\\.java:[0-9]{1,5}',
+    } as LokiExpression);
   });
+
+  it('returns query with filter operands as is', () => {
+    expect(parseQuery('{foo="bar"} |= "x|y"')).toEqual({
+      query: '{foo="bar"} |= "x|y"',
+      regexp: '',
+    } as LokiExpression);
+    expect(parseQuery('{foo="bar"} |~ "42"')).toEqual({
+      query: '{foo="bar"} |~ "42"',
+      regexp: '',
+    } as LokiExpression);
+  });
+});
+
+describe('getHighlighterExpressionsFromQuery', () => {
+  it('returns no expressions for empty query', () => {
+    expect(getHighlighterExpressionsFromQuery('')).toEqual([]);
+  });
+  it('returns a single expressions for legacy query', () => {
+    expect(getHighlighterExpressionsFromQuery('{} x')).toEqual(['(?i)x']);
+    expect(getHighlighterExpressionsFromQuery('{foo="bar"} x')).toEqual(['(?i)x']);
+  });
+  it('returns an expression for query with filter', () => {
+    expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x"')).toEqual(['x']);
+  });
+  it('returns expressions for query with filter chain', () => {
+    expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" |~ "y"')).toEqual(['x', 'y']);
+  });
+  it('returns drops expressions for query with negative filter chain', () => {
+    expect(getHighlighterExpressionsFromQuery('{foo="bar"} |= "x" != "y"')).toEqual(['x']);
   });
 });

public/app/plugins/datasource/loki/query_utils.ts

@@ -1,22 +1,70 @@
+import { LokiExpression } from './types';
+
 const selectorRegexp = /(?:^|\s){[^{]*}/g;
 const caseInsensitive = '(?i)'; // Golang mode modifier for Loki, doesn't work in JavaScript
-export function parseQuery(input: string) {
+export function parseQuery(input: string): LokiExpression {
   input = input || '';
   const match = input.match(selectorRegexp);
-  let query = '';
+  let query = input;
-  let regexp = input;
+  let regexp = '';
 
   if (match) {
-    query = match[0].trim();
     regexp = input.replace(selectorRegexp, '').trim();
+    // Keep old-style regexp, otherwise take whole query
+    if (regexp && regexp.search(/\|=|\|~|!=|!~/) === -1) {
+      query = match[0].trim();
+      if (!regexp.startsWith(caseInsensitive)) {
+        regexp = `${caseInsensitive}${regexp}`;
+      }
+    } else {
+      regexp = '';
+    }
   }
 
-  if (regexp) {
+  return { regexp, query };
-    regexp = caseInsensitive + regexp;
-  }
-  return { query, regexp };
 }
 
 export function formatQuery(selector: string, search: string): string {
   return `${selector || ''} ${search || ''}`.trim();
 }
+
+/**
+ * Returns search terms from a LogQL query.
+ * E.g., `{} |= foo |=bar != baz` returns `['foo', 'bar']`.
+ */
+export function getHighlighterExpressionsFromQuery(input: string): string[] {
+  const parsed = parseQuery(input);
+  // Legacy syntax
+  if (parsed.regexp) {
+    return [parsed.regexp];
+  }
+  let expression = input;
+  const results = [];
+  // Consume filter expression from left to right
+  while (expression) {
+    const filterStart = expression.search(/\|=|\|~|!=|!~/);
+    // Nothing more to search
+    if (filterStart === -1) {
+      break;
+    }
+    // Drop terms for negative filters
+    const skip = expression.substr(filterStart).search(/!=|!~/) === 0;
+    expression = expression.substr(filterStart + 2);
+    if (skip) {
+      continue;
+    }
+    // Check if there is more chained
+    const filterEnd = expression.search(/\|=|\|~|!=|!~/);
+    let filterTerm;
+    if (filterEnd === -1) {
+      filterTerm = expression.trim();
+    } else {
+      filterTerm = expression.substr(0, filterEnd);
+      expression = expression.substr(filterEnd);
+    }
+
+    // Unwrap the filter term by removing quotes
+    results.push(filterTerm.replace(/^\s*"/g, '').replace(/"\s*$/g, ''));
+  }
+  return results;
+}

public/app/plugins/datasource/loki/types.ts

@@ -22,3 +22,8 @@ export interface LokiLogsStreamEntry {
   // Legacy, was renamed to ts
   timestamp?: string;
 }
+
+export interface LokiExpression {
+  regexp: string;
+  query: string;
+}