Loki: LogQL v2 support (#27884)

* Add new logql v2 functions

* Fix autocompletion if missing ending }

* Refactor operators regex, add pipe operator and tests

* Add parsers

* Update tests

* Add parsers to suggestions, add test

* Add operators to syntax

* Create pipe operator autocomplete + highlighting + add tests

* Add to documentation that pipe operations are available in in Loki 2.0+

* Update snapshot test

* Update operators list, add regex quotes and move cursor

* Fix spelling

* Update documentation

* Update

public/app/plugins/datasource/loki/components/__snapshots__/LokiExploreQueryEditor.test.tsx.snap

@@ -54,6 +54,7 @@ exports[`LokiExploreQueryEditor should render component 1`] = `
         "datasource": [Circular],
         "fetchSeriesLabels": [Function],
         "getBeginningCompletionItems": [Function],
+        "getPipeCompletionItem": [Function],
         "getTermCompletionItems": [Function],
         "labelKeys": Array [],
         "labelsCache": LRUCache {

public/app/plugins/datasource/loki/language_provider.test.ts

@@ -82,6 +82,16 @@ describe('Language completion provider', () => {
       expect(result.suggestions[0].label).toEqual('History');
       expect(result.suggestions[1].label).toEqual('Functions');
     });
+
+    it('returns pipe operations on pipe context', async () => {
+      const instance = new LanguageProvider(datasource);
+      const input = createTypeaheadInput('{app="test"} | ', ' ', '', 15, ['context-pipe']);
+      const result = await instance.provideCompletionItems(input, { absoluteRange: rangeMock });
+      expect(result.context).toBeUndefined();
+      expect(result.suggestions.length).toEqual(2);
+      expect(result.suggestions[0].label).toEqual('Operators');
+      expect(result.suggestions[1].label).toEqual('Parsers');
+    });
   });
 
   describe('label key suggestions', () => {

public/app/plugins/datasource/loki/language_provider.ts

@@ -9,7 +9,7 @@ import {
   selectorRegexp,
   processLabels,
 } from 'app/plugins/datasource/prometheus/language_utils';
-import syntax, { FUNCTIONS } from './syntax';
+import syntax, { FUNCTIONS, PIPE_PARSERS, PIPE_OPERATORS } from './syntax';
 
 // Types
 import { LokiQuery } from './types';
@@ -84,7 +84,7 @@ export default class LokiLanguageProvider extends LanguageProvider {
   }
 
   // Strip syntax chars
-  cleanText = (s: string) => s.replace(/[{}[\]="(),!~+\-*/^%]/g, '').trim();
+  cleanText = (s: string) => s.replace(/[{}[\]="(),!~+\-*/^%\|]/g, '').trim();
 
   getSyntax(): Grammar {
     return syntax;
@@ -165,6 +165,8 @@ export default class LokiLanguageProvider extends LanguageProvider {
     } else if (wrapperClasses.includes('context-labels')) {
       // Suggestions for {|} and {foo=|}
       return await this.getLabelCompletionItems(input, context);
+    } else if (wrapperClasses.includes('context-pipe')) {
+      return this.getPipeCompletionItem();
     } else if (empty) {
       // Suggestions for empty query field
       return this.getEmptyCompletionItems(context);
@@ -222,6 +224,22 @@ export default class LokiLanguageProvider extends LanguageProvider {
     return { suggestions };
   };
 
+  getPipeCompletionItem = (): TypeaheadOutput => {
+    const suggestions = [];
+
+    suggestions.push({
+      label: 'Operators',
+      items: PIPE_OPERATORS.map(suggestion => ({ ...suggestion, kind: 'operators' })),
+    });
+
+    suggestions.push({
+      label: 'Parsers',
+      items: PIPE_PARSERS.map(suggestion => ({ ...suggestion, kind: 'parsers' })),
+    });
+
+    return { suggestions };
+  };
+
   getRangeCompletionItems(): TypeaheadOutput {
     return {
       context: 'context-range',

public/app/plugins/datasource/loki/syntax.test.ts

@@ -18,5 +18,32 @@ describe('Loki syntax', () => {
     expect(Prism.highlight('{key="value"}#test', syntax, 'loki')).toBe(
       '<span class="token context-labels"><span class="token punctuation">{</span><span class="token label-key attr-name">key</span>=<span class="token label-value attr-value">"value"</span></span><span class="token punctuation">}</span><span class="token comment">#test</span>'
     );
+    expect(Prism.highlight('{key="value"', syntax, 'loki')).toBe(
+      '<span class="token context-labels"><span class="token punctuation">{</span><span class="token label-key attr-name">key</span>=<span class="token label-value attr-value">"value"</span></span>'
+    );
+  });
+  it('should highlight functions in Loki query correctly', () => {
+    expect(Prism.highlight('rate({key="value"}[5m])', syntax, 'loki')).toContain(
+      '<span class="token function">rate</span>'
+    );
+    expect(Prism.highlight('avg_over_time({key="value"}[5m])', syntax, 'loki')).toContain(
+      '<span class="token function">avg_over_time</span>'
+    );
+  });
+  it('should highlight operators in Loki query correctly', () => {
+    expect(Prism.highlight('{key="value"} |= "test"', syntax, 'loki')).toContain(
+      '<span class="token operator"> |= </span>'
+    );
+    expect(Prism.highlight('{key="value"} |~"test"', syntax, 'loki')).toContain(
+      '<span class="token operator"> |~</span>'
+    );
+  });
+  it('should highlight pipe operations in Loki query correctly', () => {
+    expect(Prism.highlight('{key="value"} |= "test" | logfmt', syntax, 'loki')).toContain(
+      '<span class="token pipe-operator operator">|</span> <span class="token pipe-operations keyword">logfmt</span></span>'
+    );
+    expect(Prism.highlight('{key="value"} |= "test" | label_format', syntax, 'loki')).toContain(
+      '<span class="token context-pipe"> <span class="token pipe-operator operator">|</span> <span class="token pipe-operations keyword">label_format</span></span>'
+    );
   });
 });

public/app/plugins/datasource/loki/syntax.ts

@@ -49,13 +49,108 @@ const AGGREGATION_OPERATORS: CompletionItem[] = [
   },
 ];
 
+export const PIPE_PARSERS: CompletionItem[] = [
+  {
+    label: 'json',
+    insertText: 'json',
+    documentation: 'Extracting labels from the log line using json parser. Only available in Loki 2.0+.',
+  },
+  {
+    label: 'regexp',
+    insertText: 'regexp ""',
+    documentation: 'Extracting labels from the log line using regexp parser. Only available in Loki 2.0+.',
+    move: -1,
+  },
+  {
+    label: 'logfmt',
+    insertText: 'logfmt',
+    documentation: 'Extracting labels from the log line using logfmt parser. Only available in Loki 2.0+.',
+  },
+];
+
+export const PIPE_OPERATORS: CompletionItem[] = [
+  {
+    label: 'unwrap',
+    insertText: 'unwrap',
+    detail: 'unwrap identifier',
+    documentation:
+      'Take labels and use the values as sample data for metric aggregations. Only available in Loki 2.0+.',
+  },
+  {
+    label: 'label_format',
+    insertText: 'label_format',
+    documentation: 'Only available in Loki 2.0+.',
+  },
+  {
+    label: 'line_format',
+    insertText: 'line_format',
+    documentation: 'Only available in Loki 2.0+.',
+  },
+];
+
 export const RANGE_VEC_FUNCTIONS = [
+  {
+    insertText: 'avg_over_time',
+    label: 'avg_over_time',
+    detail: 'avg_over_time(range-vector)',
+    documentation: 'The average of all values in the specified interval. Only available in Loki 2.0+.',
+  },
+  {
+    insertText: 'min_over_time',
+    label: 'min_over_time',
+    detail: 'min_over_time(range-vector)',
+    documentation: 'The minimum of all values in the specified interval. Only available in Loki 2.0+.',
+  },
+  {
+    insertText: 'max_over_time',
+    label: 'max_over_time',
+    detail: 'max_over_time(range-vector)',
+    documentation: 'The maximum of all values in the specified interval. Only available in Loki 2.0+.',
+  },
+  {
+    insertText: 'sum_over_time',
+    label: 'sum_over_time',
+    detail: 'sum_over_time(range-vector)',
+    documentation: 'The sum of all values in the specified interval. Only available in Loki 2.0+.',
+  },
   {
     insertText: 'count_over_time',
     label: 'count_over_time',
     detail: 'count_over_time(range-vector)',
     documentation: 'The count of all values in the specified interval.',
   },
+  {
+    insertText: 'stdvar_over_time',
+    label: 'stdvar_over_time',
+    detail: 'stdvar_over_time(range-vector)',
+    documentation:
+      'The population standard variance of the values in the specified interval. Only available in Loki 2.0+.',
+  },
+  {
+    insertText: 'stddev_over_time',
+    label: 'stddev_over_time',
+    detail: 'stddev_over_time(range-vector)',
+    documentation:
+      'The population standard deviation of the values in the specified interval. Only available in Loki 2.0+.',
+  },
+  {
+    insertText: 'quantile_over_time',
+    label: 'quantile_over_time',
+    detail: 'quantile_over_time(scalar, range-vector)',
+    documentation: 'The φ-quantile (0 ≤ φ ≤ 1) of the values in the specified interval. Only available in Loki 2.0+.',
+  },
+  {
+    insertText: 'bytes_over_time',
+    label: 'bytes_over_time',
+    detail: 'bytes_over_time(range-vector)',
+    documentation: 'Counts the amount of bytes used by each log stream for a given range',
+  },
+  {
+    insertText: 'bytes_rate',
+    label: 'bytes_rate',
+    detail: 'bytes_rate(range-vector)',
+    documentation: 'Calculates the number of bytes per second for each stream.',
+  },
   {
     insertText: 'rate',
     label: 'rate',
@@ -83,7 +178,7 @@ const tokenizer: Grammar = {
     },
   },
   'context-labels': {
-    pattern: /\{[^}]*(?=})/,
+    pattern: /\{[^}]*(?=}?)/,
     greedy: true,
     inside: {
       comment: {
@@ -102,6 +197,19 @@ const tokenizer: Grammar = {
       punctuation: /[{]/,
     },
   },
+  'context-pipe': {
+    pattern: /\s\|[^=~]\s?\w*/i,
+    inside: {
+      'pipe-operator': {
+        pattern: /\|/i,
+        alias: 'operator',
+      },
+      'pipe-operations': {
+        pattern: new RegExp(`${[...PIPE_PARSERS, ...PIPE_OPERATORS].map(f => f.label).join('|')}`, 'i'),
+        alias: 'keyword',
+      },
+    },
+  },
   function: new RegExp(`\\b(?:${FUNCTIONS.map(f => f.label).join('|')})(?=\\s*\\()`, 'i'),
   'context-range': [
     {
@@ -125,7 +233,7 @@ const tokenizer: Grammar = {
     },
   ],
   number: /\b-?\d+((\.\d*)?([eE][+-]?\d+)?)?\b/,
-  operator: new RegExp(`/&&?|\\|?\\||!=?|<(?:=>?|<|>)?|>[>=]?`, 'i'),
+  operator: /\s?(\|[=~]?|!=?|<(?:=>?|<|>)?|>[>=]?)\s?/i,
   punctuation: /[{}()`,.]/,
 };