Loki: Remove distinct operation (#73938)

* remove distinct

* trigger ci

* update yarn.lock

* fix import

package.json

@@ -241,7 +241,7 @@
     "@grafana/faro-core": "1.1.2",
     "@grafana/faro-web-sdk": "1.1.2",
     "@grafana/google-sdk": "0.1.1",
-    "@grafana/lezer-logql": "0.1.9",
+    "@grafana/lezer-logql": "0.1.11",
     "@grafana/lezer-traceql": "0.0.4",
     "@grafana/monaco-logql": "^0.0.7",
     "@grafana/runtime": "workspace:*",

public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.test.ts

@@ -124,12 +124,6 @@ const afterSelectorCompletions = [
     type: 'PIPE_OPERATION',
     documentation: 'Operator docs',
   },
-  {
-    documentation: 'Operator docs',
-    insertText: '| distinct',
-    label: 'distinct',
-    type: 'PIPE_OPERATION',
-  },
   {
     documentation: 'Operator docs',
     insertText: '| drop',
@@ -400,32 +394,6 @@ describe('getCompletions', () => {
     expect(functionCompletions).toHaveLength(3);
   });
 
-  test('Returns completion options when the situation is AFTER_DISTINCT', async () => {
-    const situation: Situation = { type: 'AFTER_DISTINCT', logQuery: '{label="value"}' };
-    const completions = await getCompletions(situation, completionProvider);
-
-    expect(completions).toEqual([
-      {
-        insertText: 'extracted',
-        label: 'extracted',
-        triggerOnInsert: false,
-        type: 'LABEL_NAME',
-      },
-      {
-        insertText: 'place',
-        label: 'place',
-        triggerOnInsert: false,
-        type: 'LABEL_NAME',
-      },
-      {
-        insertText: 'source',
-        label: 'source',
-        triggerOnInsert: false,
-        type: 'LABEL_NAME',
-      },
-    ]);
-  });
-
   test('Returns completion options when the situation is AFTER_KEEP_AND_DROP', async () => {
     const situation: Situation = { type: 'AFTER_KEEP_AND_DROP', logQuery: '{label="value"}' };
     const completions = await getCompletions(situation, completionProvider);

public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.ts

@@ -286,13 +286,6 @@ export async function getAfterSelectorCompletions(
     documentation: explainOperator(LokiOperationId.Decolorize),
   });
 
-  completions.push({
-    type: 'PIPE_OPERATION',
-    label: 'distinct',
-    insertText: `${prefix}distinct`,
-    documentation: explainOperator(LokiOperationId.Distinct),
-  });
-
   completions.push({
     type: 'PIPE_OPERATION',
     label: 'drop',
@@ -359,18 +352,6 @@ async function getAfterUnwrapCompletions(
   return [...labelCompletions, ...UNWRAP_FUNCTION_COMPLETIONS];
 }
 
-async function getAfterDistinctCompletions(logQuery: string, dataProvider: CompletionDataProvider) {
-  const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery);
-  const labelCompletions: Completion[] = extractedLabelKeys.map((label) => ({
-    type: 'LABEL_NAME',
-    label,
-    insertText: label,
-    triggerOnInsert: false,
-  }));
-
-  return [...labelCompletions];
-}
-
 async function getAfterKeepAndDropCompletions(logQuery: string, dataProvider: CompletionDataProvider) {
   const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery);
   const labelCompletions: Completion[] = extractedLabelKeys.map((label) => ({
@@ -417,8 +398,6 @@ export async function getCompletions(
       return getAfterUnwrapCompletions(situation.logQuery, dataProvider);
     case 'IN_AGGREGATION':
       return [...FUNCTION_COMPLETIONS, ...AGGREGATION_COMPLETIONS];
-    case 'AFTER_DISTINCT':
-      return getAfterDistinctCompletions(situation.logQuery, dataProvider);
     case 'AFTER_KEEP_AND_DROP':
       return getAfterKeepAndDropCompletions(situation.logQuery, dataProvider);
     default:

public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.test.ts

@@ -265,18 +265,6 @@ describe('situation', () => {
     });
   });
 
-  it('identifies AFTER_DISTINCT autocomplete situations', () => {
-    assertSituation('{label="value"} | logfmt | distinct^', {
-      type: 'AFTER_DISTINCT',
-      logQuery: '{label="value"} | logfmt ',
-    });
-
-    assertSituation('{label="value"} | logfmt | distinct id,^', {
-      type: 'AFTER_DISTINCT',
-      logQuery: '{label="value"} | logfmt ',
-    });
-  });
-
   it('identifies AFTER_KEEP_AND_DROP autocomplete situations', () => {
     assertSituation('{label="value"} | logfmt | drop^', {
       type: 'AFTER_KEEP_AND_DROP',

public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.ts

@@ -20,8 +20,6 @@ import {
   LiteralExpr,
   MetricExpr,
   UnwrapExpr,
-  DistinctFilter,
-  DistinctLabel,
   DropLabelsExpr,
   KeepLabelsExpr,
   DropLabels,
@@ -132,10 +130,6 @@ export type Situation =
       type: 'AFTER_UNWRAP';
       logQuery: string;
     }
-  | {
-      type: 'AFTER_DISTINCT';
-      logQuery: string;
-    }
   | {
       type: 'AFTER_KEEP_AND_DROP';
       logQuery: string;
@@ -205,14 +199,6 @@ const RESOLVERS: Resolver[] = [
     path: [UnwrapExpr],
     fun: resolveAfterUnwrap,
   },
-  {
-    path: [ERROR_NODE_ID, DistinctFilter],
-    fun: resolveAfterDistinct,
-  },
-  {
-    path: [ERROR_NODE_ID, DistinctLabel],
-    fun: resolveAfterDistinct,
-  },
   {
     path: [ERROR_NODE_ID, DropLabelsExpr],
     fun: resolveAfterKeepAndDrop,
@@ -533,29 +519,6 @@ function resolveSelector(node: SyntaxNode, text: string, pos: number): Situation
   };
 }
 
-function resolveAfterDistinct(node: SyntaxNode, text: string, pos: number): Situation | null {
-  let logQuery = getLogQueryFromMetricsQuery(text).trim();
-
-  let distinctFilterParent: SyntaxNode | null = null;
-  let parent = node.parent;
-  while (parent !== null) {
-    if (parent.type.id === PipelineStage) {
-      distinctFilterParent = parent;
-      break;
-    }
-    parent = parent.parent;
-  }
-
-  if (distinctFilterParent?.type.id === PipelineStage) {
-    logQuery = logQuery.slice(0, distinctFilterParent.from);
-  }
-
-  return {
-    type: 'AFTER_DISTINCT',
-    logQuery,
-  };
-}
-
 function resolveAfterKeepAndDrop(node: SyntaxNode, text: string, pos: number): Situation | null {
   let logQuery = getLogQueryFromMetricsQuery(text).trim();
   let keepAndDropParent: SyntaxNode | null = null;

public/app/plugins/datasource/loki/querySplitting.test.ts

@@ -340,19 +340,6 @@ describe('runSplitQuery()', () => {
         expect(datasource.runQuery).toHaveBeenCalledTimes(1);
       });
     });
-    test('Groups queries using distinct', async () => {
-      const request = getQueryOptions<LokiQuery>({
-        targets: [
-          { expr: '{a="b"} | distinct field', refId: 'A' },
-          { expr: 'count_over_time({c="d"} | distinct something [1m])', refId: 'B' },
-        ],
-        range,
-      });
-      await expect(runSplitQuery(datasource, request)).toEmitValuesWith(() => {
-        // Queries using distinct are omitted from splitting
-        expect(datasource.runQuery).toHaveBeenCalledTimes(1);
-      });
-    });
     test('Respects maxLines of logs queries', async () => {
       const { logFrameA } = getMockFrames();
       const request = getQueryOptions<LokiQuery>({
@@ -370,18 +357,17 @@ describe('runSplitQuery()', () => {
         expect(datasource.runQuery).toHaveBeenCalledTimes(4);
       });
     });
-    test('Groups multiple queries into logs, queries, instant, and distinct', async () => {
+    test('Groups multiple queries into logs, queries, instant', async () => {
       const request = getQueryOptions<LokiQuery>({
         targets: [
           { expr: 'count_over_time({a="b"}[1m])', refId: 'A', queryType: LokiQueryType.Instant },
           { expr: '{c="d"}', refId: 'B' },
           { expr: 'count_over_time({c="d"}[1m])', refId: 'C' },
-          { expr: 'count_over_time({c="d"} | distinct id [1m])', refId: 'D' },
         ],
         range,
       });
       await expect(runSplitQuery(datasource, request)).toEmitValuesWith(() => {
-        // 3 days, 3 chunks, 3x Logs + 3x Metric + (1x Instant | Distinct), 7 requests.
+        // 3 days, 3 chunks, 3x Logs + 3x Metric + (1x Instant), 7 requests.
         expect(datasource.runQuery).toHaveBeenCalledTimes(7);
       });
     });

public/app/plugins/datasource/loki/querySplitting.ts

@@ -18,7 +18,7 @@ import { LoadingState } from '@grafana/schema';
 import { LokiDatasource } from './datasource';
 import { splitTimeRange as splitLogsTimeRange } from './logsTimeSplitting';
 import { splitTimeRange as splitMetricTimeRange } from './metricTimeSplitting';
-import { isLogsQuery, isQueryWithDistinct, isQueryWithRangeVariable } from './queryUtils';
+import { isLogsQuery, isQueryWithRangeVariable } from './queryUtils';
 import { combineResponses } from './responseUtils';
 import { trackGroupedQueries } from './tracking';
 import { LokiGroupedRequest, LokiQuery, LokiQueryType } from './types';
@@ -208,7 +208,6 @@ function getNextRequestPointers(requests: LokiGroupedRequest[], requestGroup: nu
 function querySupportsSplitting(query: LokiQuery) {
   return (
     query.queryType !== LokiQueryType.Instant &&
-    !isQueryWithDistinct(query.expr) &&
     // Queries with $__range variable should not be split because then the interpolated $__range variable is incorrect
     // because it is interpolated on the backend with the split timeRange
     !isQueryWithRangeVariable(query.expr)

public/app/plugins/datasource/loki/queryUtils.test.ts

@@ -12,7 +12,6 @@ import {
   getParserFromQuery,
   obfuscate,
   requestSupportsSplitting,
-  isQueryWithDistinct,
   isQueryWithRangeVariable,
   isQueryPipelineErrorFiltering,
   getLogQueryFromMetricsQuery,
@@ -310,18 +309,6 @@ describe('isQueryWithLabelFormat', () => {
   });
 });
 
-describe('isQueryWithDistinct', () => {
-  it('identifies queries using distinct', () => {
-    expect(isQueryWithDistinct('{job="grafana"} | distinct id')).toBe(true);
-    expect(isQueryWithDistinct('count_over_time({job="grafana"} | distinct id [1m])')).toBe(true);
-  });
-
-  it('does not return false positives', () => {
-    expect(isQueryWithDistinct('{label="distinct"} | logfmt')).toBe(false);
-    expect(isQueryWithDistinct('count_over_time({job="distinct"} | json [1m])')).toBe(false);
-  });
-});
-
 describe('isQueryWithRangeVariableDuration', () => {
   it('identifies queries using $__range variable', () => {
     expect(isQueryWithRangeVariable('rate({job="grafana"}[$__range])')).toBe(true);

public/app/plugins/datasource/loki/queryUtils.ts

@@ -17,7 +17,6 @@ import {
   MetricExpr,
   Matcher,
   Identifier,
-  Distinct,
   Range,
   formatLokiQuery,
 } from '@grafana/lezer-logql';
@@ -248,10 +247,6 @@ export function isQueryWithLineFilter(query: string): boolean {
   return isQueryWithNode(query, LineFilter);
 }
 
-export function isQueryWithDistinct(query: string): boolean {
-  return isQueryWithNode(query, Distinct);
-}
-
 export function isQueryWithRangeVariable(query: string): boolean {
   const rangeNodes = getNodesFromQuery(query, [Range]);
   for (const node of rangeNodes) {

public/app/plugins/datasource/loki/querybuilder/operations.ts

@@ -1,4 +1,3 @@
-import { LabelParamEditor } from '../../prometheus/querybuilder/components/LabelParamEditor';
 import {
   createAggregationOperation,
   createAggregationOperationWithParam,
@@ -487,27 +486,6 @@ Example: \`\`error_level=\`level\` \`\`
       addOperationHandler: addLokiOperation,
       explainHandler: () => `This will remove ANSI color codes from log lines.`,
     },
-    {
-      id: LokiOperationId.Distinct,
-      name: 'Distinct',
-      params: [
-        {
-          name: 'Label',
-          type: 'string',
-          restParam: true,
-          optional: true,
-          editor: LabelParamEditor,
-        },
-      ],
-      defaultParams: [''],
-      alternativesKey: 'format',
-      category: LokiVisualQueryOperationCategory.Formats,
-      orderRank: LokiOperationOrder.Unwrap,
-      renderer: (op, def, innerExpr) => `${innerExpr} | distinct ${op.params.join(',')}`,
-      addOperationHandler: addLokiOperation,
-      explainHandler: () =>
-        'Allows filtering log lines using their original and extracted labels to filter out duplicate label values. The first line occurrence of a distinct value is returned, and the others are dropped.',
-    },
     {
       id: LokiOperationId.Drop,
       name: 'Drop',

public/app/plugins/datasource/loki/querybuilder/parsing.test.ts

@@ -747,36 +747,6 @@ describe('buildVisualQueryFromString', () => {
     });
   });
 
-  it('parses a log query with distinct and no labels', () => {
-    expect(buildVisualQueryFromString('{app="frontend"} | distinct')).toEqual(
-      noErrors({
-        labels: [
-          {
-            op: '=',
-            value: 'frontend',
-            label: 'app',
-          },
-        ],
-        operations: [{ id: LokiOperationId.Distinct, params: [] }],
-      })
-    );
-  });
-
-  it('parses a log query with distinct and labels', () => {
-    expect(buildVisualQueryFromString('{app="frontend"} | distinct id, email')).toEqual(
-      noErrors({
-        labels: [
-          {
-            op: '=',
-            value: 'frontend',
-            label: 'app',
-          },
-        ],
-        operations: [{ id: LokiOperationId.Distinct, params: ['id', 'email'] }],
-      })
-    );
-  });
-
   it('parses a log query with drop and no labels', () => {
     expect(buildVisualQueryFromString('{app="frontend"} | drop')).toEqual(
       noErrors({

public/app/plugins/datasource/loki/querybuilder/parsing.ts

@@ -8,8 +8,6 @@ import {
   By,
   ConvOp,
   Decolorize,
-  DistinctFilter,
-  DistinctLabel,
   DropLabel,
   DropLabels,
   DropLabelsExpr,
@@ -213,11 +211,6 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex
       break;
     }
 
-    case DistinctFilter: {
-      visQuery.operations.push(handleDistinctFilter(expr, node, context));
-      break;
-    }
-
     case DropLabelsExpr: {
       visQuery.operations.push(handleDropFilter(expr, node, context));
       break;
@@ -660,23 +653,6 @@ function isEmptyQuery(query: LokiVisualQuery) {
   return false;
 }
 
-function handleDistinctFilter(expr: string, node: SyntaxNode, context: Context): QueryBuilderOperation {
-  const labels: string[] = [];
-  let exploringNode = node.getChild(DistinctLabel);
-  while (exploringNode) {
-    const label = getString(expr, exploringNode.getChild(Identifier));
-    if (label) {
-      labels.push(label);
-    }
-    exploringNode = exploringNode?.getChild(DistinctLabel);
-  }
-  labels.reverse();
-  return {
-    id: LokiOperationId.Distinct,
-    params: labels,
-  };
-}
-
 function handleDropFilter(expr: string, node: SyntaxNode, context: Context): QueryBuilderOperation {
   const labels: string[] = [];
   let exploringNode = node.getChild(DropLabels);

public/app/plugins/datasource/loki/querybuilder/types.ts

@@ -38,7 +38,6 @@ export enum LokiOperationId {
   Regexp = 'regexp',
   Pattern = 'pattern',
   Unpack = 'unpack',
-  Distinct = 'distinct',
   LineFormat = 'line_format',
   LabelFormat = 'label_format',
   Decolorize = 'decolorize',

yarn.lock

@@ -3867,14 +3867,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@grafana/lezer-logql@npm:0.1.9":
+"@grafana/lezer-logql@npm:0.1.11":
-  version: 0.1.9
+  version: 0.1.11
-  resolution: "@grafana/lezer-logql@npm:0.1.9"
+  resolution: "@grafana/lezer-logql@npm:0.1.11"
-  dependencies:
-    lodash: ^4.17.21
   peerDependencies:
     "@lezer/lr": ^1.0.0
-  checksum: 6cceb18586413864137ef2305bbe2fdce054796c61a3fde4864e3f2d30ea3dac853aa701728867bf609a4009334dbf92fddbc06f7291dbca9459a86c9dc29e67
+  checksum: 6a624b9a8d31ff854fcf9708c35e6a7498e78c4bda884639681d0b6d0fffe5527fbaeab1198e5a7694f913181657334345f31156a4a15ff64e3019b30ba6ca2a
   languageName: node
   linkType: hard
 
@@ -19255,7 +19253,7 @@ __metadata:
     "@grafana/faro-core": 1.1.2
     "@grafana/faro-web-sdk": 1.1.2
     "@grafana/google-sdk": 0.1.1
-    "@grafana/lezer-logql": 0.1.9
+    "@grafana/lezer-logql": 0.1.11
     "@grafana/lezer-traceql": 0.0.4
     "@grafana/monaco-logql": ^0.0.7
     "@grafana/runtime": "workspace:*"