Loki: Add autocomplete updates for improved suggestions (#64744)

* Loki: Add autocomplete updates for improved suggestions

* Use trimEnd for trailing pipeline

* Update public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.ts

Co-authored-by: Matias Chomicki <matyax@gmail.com>

* Fix unused imports

---------

Co-authored-by: Matias Chomicki <matyax@gmail.com>

public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.test.ts

@@ -3,7 +3,7 @@ import { LokiDatasource } from '../../../datasource';
 import { createLokiDatasource } from '../../../mocks';
 
 import { CompletionDataProvider } from './CompletionDataProvider';
-import { getCompletions } from './completions';
+import { getAfterSelectorCompletions, getCompletions } from './completions';
 import { Label, Situation } from './situation';
 
 jest.mock('../../../querybuilder/operations', () => ({
@@ -98,27 +98,6 @@ const afterSelectorCompletions = [
     label: 'unpack',
     type: 'PARSER',
   },
-  {
-    insertText: '| unwrap extracted',
-    label: 'unwrap extracted',
-    type: 'PIPE_OPERATION',
-  },
-  {
-    insertText: '| unwrap place',
-    label: 'unwrap place',
-    type: 'PIPE_OPERATION',
-  },
-  {
-    insertText: '| unwrap source',
-    label: 'unwrap source',
-    type: 'PIPE_OPERATION',
-  },
-  {
-    insertText: '| unwrap',
-    label: 'unwrap',
-    type: 'PIPE_OPERATION',
-    documentation: 'Operator docs',
-  },
   {
     insertText: '| line_format "{{.$0}}"',
     isSnippet: true,
@@ -133,6 +112,12 @@ const afterSelectorCompletions = [
     type: 'PIPE_OPERATION',
     documentation: 'Operator docs',
   },
+  {
+    insertText: '| unwrap',
+    label: 'unwrap',
+    type: 'PIPE_OPERATION',
+    documentation: 'Operator docs',
+  },
 ];
 
 function buildAfterSelectorCompletions(
@@ -333,7 +318,7 @@ describe('getCompletions', () => {
         hasJSON: true,
         hasLogfmt: false,
       });
-      const situation: Situation = { type: 'AFTER_SELECTOR', logQuery: '', afterPipe, hasSpace };
+      const situation: Situation = { type: 'AFTER_SELECTOR', logQuery: '{job="grafana"}', afterPipe, hasSpace };
       const completions = await getCompletions(situation, completionProvider);
 
       const expected = buildAfterSelectorCompletions('json', 'logfmt', afterPipe, hasSpace);
@@ -389,3 +374,78 @@ describe('getCompletions', () => {
     expect(functionCompletions).toHaveLength(3);
   });
 });
+
+describe('getAfterSelectorCompletions', () => {
+  let datasource: LokiDatasource;
+  let languageProvider: LokiLanguageProvider;
+  let completionProvider: CompletionDataProvider;
+
+  beforeEach(() => {
+    datasource = createLokiDatasource();
+    languageProvider = new LokiLanguageProvider(datasource);
+    completionProvider = new CompletionDataProvider(languageProvider, {
+      current: history,
+    });
+
+    jest.spyOn(completionProvider, 'getParserAndLabelKeys').mockResolvedValue({
+      extractedLabelKeys: ['abc', 'def'],
+      unwrapLabelKeys: [],
+      hasJSON: true,
+      hasLogfmt: false,
+    });
+  });
+  it('should remove trailing pipeline from logQuery', () => {
+    getAfterSelectorCompletions(`{job="grafana"} | `, true, true, completionProvider);
+    expect(completionProvider.getParserAndLabelKeys).toHaveBeenCalledWith(`{job="grafana"}`);
+  });
+
+  it('should show detected parser if query has no parser', async () => {
+    const suggestions = await getAfterSelectorCompletions(`{job="grafana"} |  `, true, true, completionProvider);
+    const parsersInSuggestions = suggestions
+      .filter((suggestion) => suggestion.type === 'PARSER')
+      .map((parser) => parser.label);
+    expect(parsersInSuggestions).toStrictEqual(['json (detected)', 'logfmt', 'pattern', 'regexp', 'unpack']);
+  });
+
+  it('should not show detected parser if query already has parser', async () => {
+    const suggestions = await getAfterSelectorCompletions(
+      `{job="grafana"} | logfmt | `,
+      true,
+      true,
+      completionProvider
+    );
+    const parsersInSuggestions = suggestions
+      .filter((suggestion) => suggestion.type === 'PARSER')
+      .map((parser) => parser.label);
+    expect(parsersInSuggestions).toStrictEqual(['json', 'logfmt', 'pattern', 'regexp', 'unpack']);
+  });
+
+  it('should show label filter options if query has parser and trailing pipeline', async () => {
+    const suggestions = await getAfterSelectorCompletions(
+      `{job="grafana"} | logfmt | `,
+      true,
+      true,
+      completionProvider
+    );
+    const labelFiltersInSuggestions = suggestions
+      .filter((suggestion) => suggestion.type === 'LABEL_NAME')
+      .map((label) => label.label);
+    expect(labelFiltersInSuggestions).toStrictEqual(['abc (detected)', 'def (detected)']);
+  });
+
+  it('should show label filter options if query has parser and no trailing pipeline', async () => {
+    const suggestions = await getAfterSelectorCompletions(`{job="grafana"} | logfmt`, true, true, completionProvider);
+    const labelFiltersInSuggestions = suggestions
+      .filter((suggestion) => suggestion.type === 'LABEL_NAME')
+      .map((label) => label.label);
+    expect(labelFiltersInSuggestions).toStrictEqual(['abc (detected)', 'def (detected)']);
+  });
+
+  it('should not show label filter options if query has no parser', async () => {
+    const suggestions = await getAfterSelectorCompletions(`{job="grafana"} | `, true, true, completionProvider);
+    const labelFiltersInSuggestions = suggestions
+      .filter((suggestion) => suggestion.type === 'LABEL_NAME')
+      .map((label) => label.label);
+    expect(labelFiltersInSuggestions.length).toBe(0);
+  });
+});

public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.ts

@@ -1,4 +1,7 @@
+import { trimEnd } from 'lodash';
+
 import { escapeLabelValueInExactSelector } from '../../../languageUtils';
+import { isQueryWithParser } from '../../../queryUtils';
 import { explainOperator } from '../../../querybuilder/operations';
 import { LokiOperationId } from '../../../querybuilder/types';
 import { AGGREGATION_OPERATORS, RANGE_VEC_FUNCTIONS, BUILT_IN_FUNCTIONS } from '../../../syntax';
@@ -171,15 +174,18 @@ async function getParserCompletions(
   prefix: string,
   hasJSON: boolean,
   hasLogfmt: boolean,
-  extractedLabelKeys: string[]
+  extractedLabelKeys: string[],
+  hasParserInQuery: boolean
 ) {
   const allParsers = new Set(PARSERS);
   const completions: Completion[] = [];
+  // We use this to improve documentation specifically for level label as it is tied to showing color-coded logs volume
   const hasLevelInExtractedLabels = extractedLabelKeys.some((key) => key === 'level');
 
   if (hasJSON) {
     allParsers.delete('json');
-    const extra = hasLevelInExtractedLabels ? '' : ' (detected)';
+    // We show "detected" label only if there is no previous parser in the query
+    const extra = hasParserInQuery ? '' : ' (detected)';
     completions.push({
       type: 'PARSER',
       label: `json${extra}`,
@@ -192,7 +198,8 @@ async function getParserCompletions(
 
   if (hasLogfmt) {
     allParsers.delete('logfmt');
-    const extra = hasLevelInExtractedLabels ? '' : ' (detected)';
+    // We show "detected" label only if there is no previous parser in the query
+    const extra = hasParserInQuery ? '' : ' (detected)';
     completions.push({
       type: 'PARSER',
       label: `logfmt${extra}`,
@@ -216,31 +223,28 @@ async function getParserCompletions(
   return completions;
 }
 
-async function getAfterSelectorCompletions(
+export async function getAfterSelectorCompletions(
   logQuery: string,
   afterPipe: boolean,
   hasSpace: boolean,
   dataProvider: CompletionDataProvider
 ): Promise<Completion[]> {
-  const { extractedLabelKeys, hasJSON, hasLogfmt } = await dataProvider.getParserAndLabelKeys(logQuery);
+  let query = logQuery;
+  if (afterPipe) {
+    query = trimEnd(logQuery, '| ');
+  }
+
+  const { extractedLabelKeys, hasJSON, hasLogfmt } = await dataProvider.getParserAndLabelKeys(query);
+  const hasQueryParser = isQueryWithParser(query).queryWithParser;
 
   const prefix = `${hasSpace ? '' : ' '}${afterPipe ? '' : '| '}`;
-  const completions: Completion[] = await getParserCompletions(prefix, hasJSON, hasLogfmt, extractedLabelKeys);
-
-  extractedLabelKeys.forEach((key) => {
-    completions.push({
-      type: 'PIPE_OPERATION',
-      label: `unwrap ${key}`,
-      insertText: `${prefix}unwrap ${key}`,
-    });
-  });
-
-  completions.push({
-    type: 'PIPE_OPERATION',
-    label: 'unwrap',
-    insertText: `${prefix}unwrap`,
-    documentation: explainOperator(LokiOperationId.Unwrap),
-  });
+  const completions: Completion[] = await getParserCompletions(
+    prefix,
+    hasJSON,
+    hasLogfmt,
+    extractedLabelKeys,
+    hasQueryParser
+  );
 
   completions.push({
     type: 'PIPE_OPERATION',
@@ -258,10 +262,32 @@ async function getAfterSelectorCompletions(
     documentation: explainOperator(LokiOperationId.LabelFormat),
   });
 
+  completions.push({
+    type: 'PIPE_OPERATION',
+    label: 'unwrap',
+    insertText: `${prefix}unwrap`,
+    documentation: explainOperator(LokiOperationId.Unwrap),
+  });
+
+  // Let's show label options only if query has parser
+  if (hasQueryParser) {
+    extractedLabelKeys.forEach((key) => {
+      completions.push({
+        type: 'LABEL_NAME',
+        label: `${key} (detected)`,
+        insertText: `${prefix}${key}`,
+        documentation: `"${key}" was suggested based on the content of your log lines for the label filter expression.`,
+      });
+    });
+  }
+
+  // If we have parser, we don't need to consider line filters
+  if (hasQueryParser) {
+    return [...completions];
+  }
   // With a space between the pipe and the cursor, we omit line filters
   // E.g. `{label="value"} | `
   const lineFilters = afterPipe && hasSpace ? [] : getLineFilterCompletions(afterPipe);
-
   return [...lineFilters, ...completions];
 }
 

public/app/plugins/datasource/loki/modifyQuery.ts

@@ -22,7 +22,7 @@ import { unescapeLabelValue } from './languageUtils';
 import { LokiQueryModeller } from './querybuilder/LokiQueryModeller';
 import { buildVisualQueryFromString } from './querybuilder/parsing';
 
-type Position = { from: number; to: number };
+export type Position = { from: number; to: number };
 /**
  * Adds label filter to existing query. Useful for query modification for example for ad hoc filters.
  *