Loki Query Editor: Add support for new logfmt features (#74619)

* Loki autocomplete: add IN_LOGFMT situation for log queries

* Loki autocomplete: add IN_LOGFMT situation for metric queries

* Loki autocomplete: improve handling of trailing pipes and spaces

* Loki autocomplete: add logfmt arguments completion

* Loki autocomplete: add flags support to IN_LOGFMT

* Loki autocomplete: extend IN_LOGFMT situation with labels and flag

* Loki autocomplete: return logQuery in IN_LOGFMT situation

* Loki autocomplete: offer label completions when IN_LOGFMT

* Query utils: update parser detection method

* Validation: update test

* Loki autocomplete: improve IN_LOGFMT detection when in metric query

* Loki autocomplete: improve logfmt suggestions

* Loki autocomplete: improve logfmt suggestions in different scenarios

* Loki autocomplete situation: refactor resolvers to support multiple paths

* Situation: add test case

* Loki autocomplete: allow user to use 2 flags

* Situation: change flag to flags

* Remove console log

* Validation: import test parser

* Completions: better handling of trailing comma scenario

* Upgrade lezer-logql

* Revert temporary imports

* Loki Query Builder: Add support for new logfmt features (#74858)

* Query builder: add params to logfmt definition

* Logfmt operation: add default params

* Query builder: update deprecated JsonExpression

* Operation utils: update logfmt renderer

* Query builder: parse LogfmtParser

* Query builder: parse LogfmtExpressionParser

* Remove console log

* Remove unused variable

* Remove extra character from render

* Update unit tests

* Fix unit tests

* Operations: remove restParams from logfmt booleans

* Parsing: group cases

* Formatting

* Formatting

* Update modifyQuery

* LogContextProvider: update with parser changes

* LogContextProvider: remove unnecessary type castings

It takes more energy to write `as unknow as LokiQuery` than to write a refId.

* Formatting

* Situation: use charAt instead of substring with endsWith

* Situation: explain logfmt suggestions

* Logfmt: improve flag suggestions

* Remove console log

* Completions: update test

public/app/plugins/datasource/loki/querybuilder/parsing.ts

@@ -20,16 +20,18 @@ import {
   Ip,
   IpLabelFilter,
   Json,
-  JsonExpression,
   JsonExpressionParser,
   KeepLabel,
   KeepLabels,
   KeepLabelsExpr,
+  LabelExtractionExpression,
   LabelFilter,
   LabelFormatMatcher,
   LabelParser,
   LineFilter,
   LineFormatExpr,
+  LogfmtExpressionParser,
+  LogfmtParser,
   LogRangeExpr,
   Matcher,
   MetricExpr,
@@ -37,6 +39,7 @@ import {
   On,
   Or,
   parser,
+  ParserFlag,
   Range,
   RangeAggregationExpr,
   RangeOp,
@@ -80,6 +83,11 @@ interface ParsingError {
   parentType?: string;
 }
 
+interface GetOperationResult {
+  operation?: QueryBuilderOperation;
+  error?: string;
+}
+
 export function buildVisualQueryFromString(expr: string): Context {
   const replacedExpr = replaceVariables(expr);
   const tree = parser.parse(replacedExpr);
@@ -160,6 +168,18 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex
       break;
     }
 
+    case LogfmtParser:
+    case LogfmtExpressionParser: {
+      const { operation, error } = getLogfmtParser(expr, node);
+      if (operation) {
+        visQuery.operations.push(operation);
+      }
+      if (error) {
+        context.errors.push(createNotSupportedError(expr, node, error));
+      }
+      break;
+    }
+
     case LineFormatExpr: {
       visQuery.operations.push(getLineFormat(expr, node));
       break;
@@ -250,7 +270,7 @@ function getLabel(expr: string, node: SyntaxNode): QueryBuilderLabelFilter {
   };
 }
 
-function getLineFilter(expr: string, node: SyntaxNode): { operation?: QueryBuilderOperation; error?: string } {
+function getLineFilter(expr: string, node: SyntaxNode): GetOperationResult {
   const filter = getString(expr, node.getChild(Filter));
   const filterExpr = handleQuotes(getString(expr, node.getChild(String)));
   const ipLineFilter = node.getChild(FilterOp)?.getChild(Ip);
@@ -299,14 +319,43 @@ function getJsonExpressionParser(expr: string, node: SyntaxNode): QueryBuilderOp
   const parserNode = node.getChild(Json);
   const parser = getString(expr, parserNode);
 
-  const params = [...getAllByType(expr, node, JsonExpression)];
+  const params = [...getAllByType(expr, node, LabelExtractionExpression)];
   return {
     id: parser,
     params,
   };
 }
 
-function getLabelFilter(expr: string, node: SyntaxNode): { operation?: QueryBuilderOperation; error?: string } {
+function getLogfmtParser(expr: string, node: SyntaxNode): GetOperationResult {
+  const flags: string[] = [];
+  const labels: string[] = [];
+  let error: string | undefined = undefined;
+
+  const offset = node.from;
+  node.toTree().iterate({
+    enter: (subNode) => {
+      if (subNode.type.id === ParserFlag) {
+        flags.push(expr.substring(subNode.from + offset, subNode.to + offset));
+      } else if (subNode.type.id === LabelExtractionExpression) {
+        labels.push(expr.substring(subNode.from + offset, subNode.to + offset));
+      } else if (subNode.type.id === ErrorId) {
+        error = `Unexpected string "${expr.substring(subNode.from + offset, subNode.to + offset)}"`;
+      }
+    },
+  });
+
+  const operation = {
+    id: LokiOperationId.Logfmt,
+    params: [flags.includes('--strict'), flags.includes('--keep-empty'), ...labels],
+  };
+
+  return {
+    operation,
+    error,
+  };
+}
+
+function getLabelFilter(expr: string, node: SyntaxNode): GetOperationResult {
   // Check for nodes not supported in visual builder and return error
   if (node.getChild(Or) || node.getChild(And) || node.getChild('Comma')) {
     return {
@@ -399,11 +448,7 @@ function getDecolorize(): QueryBuilderOperation {
   };
 }
 
-function handleUnwrapExpr(
-  expr: string,
-  node: SyntaxNode,
-  context: Context
-): { operation?: QueryBuilderOperation; error?: string } {
+function handleUnwrapExpr(expr: string, node: SyntaxNode, context: Context): GetOperationResult {
   const unwrapExprChild = node.getChild(UnwrapExpr);
   const labelFilterChild = node.getChild(LabelFilter);
   const unwrapChild = node.getChild(Unwrap);