Loki: Add all supported parsers to query builder (#47304)

* Loki: Add all supported parsers to query builderr

* Update test

* Add tests for new parsers

public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.test.ts

@@ -31,6 +31,33 @@ describe('LokiQueryModeller', () => {
     ).toBe('{app="grafana"} | logfmt');
   });
 
+  it('Can query with pipeline operation regexp', () => {
+    expect(
+      modeller.renderQuery({
+        labels: [{ label: 'app', op: '=', value: 'grafana' }],
+        operations: [{ id: LokiOperationId.Regexp, params: ['re'] }],
+      })
+    ).toBe('{app="grafana"} | regexp `re`');
+  });
+
+  it('Can query with pipeline operation pattern', () => {
+    expect(
+      modeller.renderQuery({
+        labels: [{ label: 'app', op: '=', value: 'grafana' }],
+        operations: [{ id: LokiOperationId.Pattern, params: ['<pattern>'] }],
+      })
+    ).toBe('{app="grafana"} | pattern `<pattern>`');
+  });
+
+  it('Can query with pipeline operation unpack', () => {
+    expect(
+      modeller.renderQuery({
+        labels: [{ label: 'app', op: '=', value: 'grafana' }],
+        operations: [{ id: LokiOperationId.Unpack, params: [] }],
+      })
+    ).toBe('{app="grafana"} | unpack');
+  });
+
   it('Can query with line filter contains operation', () => {
     expect(
       modeller.renderQuery({
@@ -82,7 +109,7 @@ describe('LokiQueryModeller', () => {
         labels: [{ label: 'app', op: '=', value: 'grafana' }],
         operations: [{ id: LokiOperationId.LabelFilter, params: ['__error__', '=', 'value'] }],
       })
-    ).toBe('{app="grafana"} | __error__="value"');
+    ).toBe('{app="grafana"} | __error__=`value`');
   });
 
   it('Can query with label filter expression using greater than operator', () => {
@@ -100,7 +127,7 @@ describe('LokiQueryModeller', () => {
         labels: [{ label: 'app', op: '=', value: 'grafana' }],
         operations: [{ id: LokiOperationId.LabelFilterNoErrors, params: [] }],
       })
-    ).toBe('{app="grafana"} | __error__=""');
+    ).toBe('{app="grafana"} | __error__=``');
   });
 
   it('Can query with unwrap operation', () => {
@@ -118,7 +145,7 @@ describe('LokiQueryModeller', () => {
         labels: [{ label: 'app', op: '=', value: 'grafana' }],
         operations: [{ id: LokiOperationId.LineFormat, params: ['{{.status_code}}'] }],
       })
-    ).toBe('{app="grafana"} | line_format "{{.status_code}}"');
+    ).toBe('{app="grafana"} | line_format `{{.status_code}}`');
   });
 
   it('Can render with label_format operation', () => {

public/app/plugins/datasource/loki/querybuilder/LokiQueryModeller.ts

@@ -1,11 +1,11 @@
 import { LokiAndPromQueryModellerBase } from '../../prometheus/querybuilder/shared/LokiAndPromQueryModellerBase';
 import { QueryBuilderLabelFilter } from '../../prometheus/querybuilder/shared/types';
-import { getOperationDefintions } from './operations';
+import { getOperationDefinitions } from './operations';
 import { LokiOperationId, LokiQueryPattern, LokiVisualQueryOperationCategory } from './types';
 
 export class LokiQueryModeller extends LokiAndPromQueryModellerBase {
   constructor() {
-    super(getOperationDefintions);
+    super(getOperationDefinitions);
 
     this.setOperationCategories([
       LokiVisualQueryOperationCategory.Aggregations,

public/app/plugins/datasource/loki/querybuilder/operations.ts

@@ -10,7 +10,7 @@ import { FUNCTIONS } from '../syntax';
 import { binaryScalarOperations } from './binaryScalarOperations';
 import { LokiOperationId, LokiOperationOrder, LokiVisualQuery, LokiVisualQueryOperationCategory } from './types';
 
-export function getOperationDefintions(): QueryBuilderOperationDef[] {
+export function getOperationDefinitions(): QueryBuilderOperationDef[] {
   const aggregations = [
     LokiOperationId.Sum,
     LokiOperationId.Min,
@@ -55,7 +55,64 @@ export function getOperationDefintions(): QueryBuilderOperationDef[] {
       renderer: pipelineRenderer,
       addOperationHandler: addLokiOperation,
       explainHandler: () =>
-        `This will extract all keys and values from a [logfmt](https://grafana.com/docs/loki/latest/logql/log_queries/#logfmt) formatted log line as labels. The extracted lables can be used in label filter expressions and used as values for a range aggregation via the unwrap operation. `,
+        `This will extract all keys and values from a [logfmt](https://grafana.com/docs/loki/latest/logql/log_queries/#logfmt) formatted log line as labels. The extracted labels can be used in label filter expressions and used as values for a range aggregation via the unwrap operation.`,
+    },
+    {
+      id: LokiOperationId.Regexp,
+      name: 'Regexp',
+      params: [
+        {
+          name: 'String',
+          type: 'string',
+          hideName: true,
+          placeholder: '<re>',
+          description: 'The regexp expression that matches the structure of a log line.',
+          minWidth: 20,
+        },
+      ],
+      defaultParams: [''],
+      alternativesKey: 'format',
+      category: LokiVisualQueryOperationCategory.Formats,
+      orderRank: LokiOperationOrder.LineFormats,
+      renderer: (model, def, innerExpr) => `${innerExpr} | regexp \`${model.params[0]}\``,
+      addOperationHandler: addLokiOperation,
+      explainHandler: () =>
+        `The [regexp parser](https://grafana.com/docs/loki/latest/logql/log_queries/#regular-expression) takes a single parameter | regexp "<re>" which is the regular expression using the Golang RE2 syntax. The regular expression must contain a least one named sub-match (e.g (?P<name>re)), each sub-match will extract a different label. The expression matches the structure of a log line. The extracted labels can be used in label filter expressions and used as values for a range aggregation via the unwrap operation.`,
+    },
+    {
+      id: LokiOperationId.Pattern,
+      name: 'Pattern',
+      params: [
+        {
+          name: 'String',
+          type: 'string',
+          hideName: true,
+          placeholder: '<pattern-expression>',
+          description: 'The expression that matches the structure of a log line.',
+          minWidth: 20,
+        },
+      ],
+      defaultParams: [''],
+      alternativesKey: 'format',
+      category: LokiVisualQueryOperationCategory.Formats,
+      orderRank: LokiOperationOrder.LineFormats,
+      renderer: (model, def, innerExpr) => `${innerExpr} | pattern \`${model.params[0]}\``,
+      addOperationHandler: addLokiOperation,
+      explainHandler: () =>
+        `The [pattern parser](https://grafana.com/docs/loki/latest/logql/log_queries/#pattern) allows the explicit extraction of fields from log lines by defining a pattern expression (| pattern \`<pattern-expression>\`). The expression matches the structure of a log line. The extracted labels can be used in label filter expressions and used as values for a range aggregation via the unwrap operation.`,
+    },
+    {
+      id: LokiOperationId.Unpack,
+      name: 'Unpack',
+      params: [],
+      defaultParams: [],
+      alternativesKey: 'format',
+      category: LokiVisualQueryOperationCategory.Formats,
+      orderRank: LokiOperationOrder.LineFormats,
+      renderer: pipelineRenderer,
+      addOperationHandler: addLokiOperation,
+      explainHandler: () =>
+        `This will extract all keys and values from a JSON log line, [unpacking](https://grafana.com/docs/loki/latest/logql/log_queries/#unpack) all embedded labels in the pack stage. The extracted labels can be used in label filter expressions and used as values for a range aggregation via the unwrap operation.`,
     },
     {
       id: LokiOperationId.LineFormat,
@@ -74,7 +131,7 @@ export function getOperationDefintions(): QueryBuilderOperationDef[] {
       alternativesKey: 'format',
       category: LokiVisualQueryOperationCategory.Formats,
       orderRank: LokiOperationOrder.LineFormats,
-      renderer: (model, def, innerExpr) => `${innerExpr} | line_format "${model.params[0]}"`,
+      renderer: (model, def, innerExpr) => `${innerExpr} | line_format \`${model.params[0]}\``,
       addOperationHandler: addLokiOperation,
       explainHandler: () =>
         `This will replace log line using a specified template. The template can refer to stream labels and extracted labels. 
@@ -212,7 +269,7 @@ export function getOperationDefintions(): QueryBuilderOperationDef[] {
       defaultParams: [],
       category: LokiVisualQueryOperationCategory.LabelFilters,
       orderRank: LokiOperationOrder.NoErrors,
-      renderer: (model, def, innerExpr) => `${innerExpr} | __error__=""`,
+      renderer: (model, def, innerExpr) => `${innerExpr} | __error__=\`\``,
       addOperationHandler: addLokiOperation,
       explainHandler: () => `Filter out all formatting and parsing errors.`,
     },
@@ -303,7 +360,7 @@ function labelFilterRenderer(model: QueryBuilderOperation, def: QueryBuilderOper
     return `${innerExpr} | ${model.params[0]} ${model.params[1]} ${model.params[2]}`;
   }
 
-  return `${innerExpr} | ${model.params[0]}${model.params[1]}"${model.params[2]}"`;
+  return `${innerExpr} | ${model.params[0]}${model.params[1]}\`${model.params[2]}\``;
 }
 
 function pipelineRenderer(model: QueryBuilderOperation, def: QueryBuilderOperationDef, innerExpr: string) {

public/app/plugins/datasource/loki/querybuilder/parsing.ts

@@ -205,8 +205,7 @@ function getLabelFilter(expr: string, node: SyntaxNode): QueryBuilderOperation {
     const label = filter!.firstChild;
     const op = label!.nextSibling;
     const value = op!.nextSibling;
-    const params = [getString(expr, label), getString(expr, op), getString(expr, value).replace(/"/g, '')];
+    const params = [getString(expr, label), getString(expr, op), handleQuotes(getString(expr, value))];
-
     //Special case of pipe filtering - no errors
     if (params.join('') === `__error__=`) {
       return {

public/app/plugins/datasource/loki/querybuilder/types.ts

@@ -30,6 +30,9 @@ export enum LokiVisualQueryOperationCategory {
 export enum LokiOperationId {
   Json = 'json',
   Logfmt = 'logfmt',
+  Regexp = 'regexp',
+  Pattern = 'pattern',
+  Unpack = 'unpack',
   LineFormat = 'line_format',
   LabelFormat = 'label_format',
   Rate = 'rate',