IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Loki: Remove distinct operation (#73938)

Browse Source 
* remove distinct

* trigger ci

* update yarn.lock

* fix import
This commit is contained in:
Sven Grossmann 2023-08-28 19:26:59 +02:00 committed by GitHub
parent 9b9c9e83dc
commit 07eb4b1b90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 9 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
package.json
View File

@ -241,7 +241,7 @@
"@grafana/faro-core": "1.1.2",
"@grafana/faro-web-sdk": "1.1.2",
"@grafana/google-sdk": "0.1.1",
"@grafana/lezer-logql": "0.1.9",
"@grafana/lezer-logql": "0.1.11",
"@grafana/lezer-traceql": "0.0.4",
"@grafana/monaco-logql": "^0.0.7",
"@grafana/runtime": "workspace:*",

32
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.test.ts
View File

@ -124,12 +124,6 @@ const afterSelectorCompletions = [
type: 'PIPE_OPERATION',
documentation: 'Operator docs',
},
{
documentation: 'Operator docs',
insertText: '| distinct',
label: 'distinct',
type: 'PIPE_OPERATION',
},
{
documentation: 'Operator docs',
insertText: '| drop',
@ -400,32 +394,6 @@ describe('getCompletions', () => {
expect(functionCompletions).toHaveLength(3);
});
test('Returns completion options when the situation is AFTER_DISTINCT', async () => {
const situation: Situation = { type: 'AFTER_DISTINCT', logQuery: '{label="value"}' };
const completions = await getCompletions(situation, completionProvider);
expect(completions).toEqual([
{
insertText: 'extracted',
label: 'extracted',
triggerOnInsert: false,
type: 'LABEL_NAME',
},
{
insertText: 'place',
label: 'place',
triggerOnInsert: false,
type: 'LABEL_NAME',
},
{
insertText: 'source',
label: 'source',
triggerOnInsert: false,
type: 'LABEL_NAME',
},
]);
});
test('Returns completion options when the situation is AFTER_KEEP_AND_DROP', async () => {
const situation: Situation = { type: 'AFTER_KEEP_AND_DROP', logQuery: '{label="value"}' };
const completions = await getCompletions(situation, completionProvider);

21
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/completions.ts
View File

@ -286,13 +286,6 @@ export async function getAfterSelectorCompletions(
documentation: explainOperator(LokiOperationId.Decolorize),
});
completions.push({
type: 'PIPE_OPERATION',
label: 'distinct',
insertText: `${prefix}distinct`,
documentation: explainOperator(LokiOperationId.Distinct),
});
completions.push({
type: 'PIPE_OPERATION',
label: 'drop',
@ -359,18 +352,6 @@ async function getAfterUnwrapCompletions(
return [...labelCompletions, ...UNWRAP_FUNCTION_COMPLETIONS];
}
async function getAfterDistinctCompletions(logQuery: string, dataProvider: CompletionDataProvider) {
const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery);
const labelCompletions: Completion[] = extractedLabelKeys.map((label) => ({
type: 'LABEL_NAME',
label,
insertText: label,
triggerOnInsert: false,
}));
return [...labelCompletions];
}
async function getAfterKeepAndDropCompletions(logQuery: string, dataProvider: CompletionDataProvider) {
const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery);
const labelCompletions: Completion[] = extractedLabelKeys.map((label) => ({
@ -417,8 +398,6 @@ export async function getCompletions(
return getAfterUnwrapCompletions(situation.logQuery, dataProvider);
case 'IN_AGGREGATION':
return [...FUNCTION_COMPLETIONS, ...AGGREGATION_COMPLETIONS];
case 'AFTER_DISTINCT':
return getAfterDistinctCompletions(situation.logQuery, dataProvider);
case 'AFTER_KEEP_AND_DROP':
return getAfterKeepAndDropCompletions(situation.logQuery, dataProvider);
default:

12
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.test.ts
View File

@ -265,18 +265,6 @@ describe('situation', () => {
});
});
it('identifies AFTER_DISTINCT autocomplete situations', () => {
assertSituation('{label="value"} | logfmt | distinct^', {
type: 'AFTER_DISTINCT',
logQuery: '{label="value"} | logfmt ',
});
assertSituation('{label="value"} | logfmt | distinct id,^', {
type: 'AFTER_DISTINCT',
logQuery: '{label="value"} | logfmt ',
});
});
it('identifies AFTER_KEEP_AND_DROP autocomplete situations', () => {
assertSituation('{label="value"} | logfmt | drop^', {
type: 'AFTER_KEEP_AND_DROP',

37
public/app/plugins/datasource/loki/components/monaco-query-field/monaco-completion-provider/situation.ts
View File

@ -20,8 +20,6 @@ import {
LiteralExpr,
MetricExpr,
UnwrapExpr,
DistinctFilter,
DistinctLabel,
DropLabelsExpr,
KeepLabelsExpr,
DropLabels,
@ -132,10 +130,6 @@ export type Situation =
type: 'AFTER_UNWRAP';
logQuery: string;
}
| {
type: 'AFTER_DISTINCT';
logQuery: string;
}
| {
type: 'AFTER_KEEP_AND_DROP';
logQuery: string;
@ -205,14 +199,6 @@ const RESOLVERS: Resolver[] = [
path: [UnwrapExpr],
fun: resolveAfterUnwrap,
},
{
path: [ERROR_NODE_ID, DistinctFilter],
fun: resolveAfterDistinct,
},
{
path: [ERROR_NODE_ID, DistinctLabel],
fun: resolveAfterDistinct,
},
{
path: [ERROR_NODE_ID, DropLabelsExpr],
fun: resolveAfterKeepAndDrop,
@ -533,29 +519,6 @@ function resolveSelector(node: SyntaxNode, text: string, pos: number): Situation
};
}
function resolveAfterDistinct(node: SyntaxNode, text: string, pos: number): Situation | null {
let logQuery = getLogQueryFromMetricsQuery(text).trim();
let distinctFilterParent: SyntaxNode | null = null;
let parent = node.parent;
while (parent !== null) {
if (parent.type.id === PipelineStage) {
distinctFilterParent = parent;
break;
}
parent = parent.parent;
}
if (distinctFilterParent?.type.id === PipelineStage) {
logQuery = logQuery.slice(0, distinctFilterParent.from);
}
return {
type: 'AFTER_DISTINCT',
logQuery,
};
}
function resolveAfterKeepAndDrop(node: SyntaxNode, text: string, pos: number): Situation | null {
let logQuery = getLogQueryFromMetricsQuery(text).trim();
let keepAndDropParent: SyntaxNode | null = null;

18
public/app/plugins/datasource/loki/querySplitting.test.ts
View File

@ -340,19 +340,6 @@ describe('runSplitQuery()', () => {
expect(datasource.runQuery).toHaveBeenCalledTimes(1);
});
});
test('Groups queries using distinct', async () => {
const request = getQueryOptions<LokiQuery>({
targets: [
{ expr: '{a="b"} | distinct field', refId: 'A' },
{ expr: 'count_over_time({c="d"} | distinct something [1m])', refId: 'B' },
],
range,
});
await expect(runSplitQuery(datasource, request)).toEmitValuesWith(() => {
// Queries using distinct are omitted from splitting
expect(datasource.runQuery).toHaveBeenCalledTimes(1);
});
});
test('Respects maxLines of logs queries', async () => {
const { logFrameA } = getMockFrames();
const request = getQueryOptions<LokiQuery>({
@ -370,18 +357,17 @@ describe('runSplitQuery()', () => {
expect(datasource.runQuery).toHaveBeenCalledTimes(4);
});
});
test('Groups multiple queries into logs, queries, instant, and distinct', async () => {
test('Groups multiple queries into logs, queries, instant', async () => {
const request = getQueryOptions<LokiQuery>({
targets: [
{ expr: 'count_over_time({a="b"}[1m])', refId: 'A', queryType: LokiQueryType.Instant },
{ expr: '{c="d"}', refId: 'B' },
{ expr: 'count_over_time({c="d"}[1m])', refId: 'C' },
{ expr: 'count_over_time({c="d"} | distinct id [1m])', refId: 'D' },
],
range,
});
await expect(runSplitQuery(datasource, request)).toEmitValuesWith(() => {
// 3 days, 3 chunks, 3x Logs + 3x Metric + (1x Instant | Distinct), 7 requests.
// 3 days, 3 chunks, 3x Logs + 3x Metric + (1x Instant), 7 requests.
expect(datasource.runQuery).toHaveBeenCalledTimes(7);
});
});

3
public/app/plugins/datasource/loki/querySplitting.ts
View File

@ -18,7 +18,7 @@ import { LoadingState } from '@grafana/schema';
import { LokiDatasource } from './datasource';
import { splitTimeRange as splitLogsTimeRange } from './logsTimeSplitting';
import { splitTimeRange as splitMetricTimeRange } from './metricTimeSplitting';
import { isLogsQuery, isQueryWithDistinct, isQueryWithRangeVariable } from './queryUtils';
import { isLogsQuery, isQueryWithRangeVariable } from './queryUtils';
import { combineResponses } from './responseUtils';
import { trackGroupedQueries } from './tracking';
import { LokiGroupedRequest, LokiQuery, LokiQueryType } from './types';
@ -208,7 +208,6 @@ function getNextRequestPointers(requests: LokiGroupedRequest[], requestGroup: nu
function querySupportsSplitting(query: LokiQuery) {
return (
query.queryType !== LokiQueryType.Instant &&
!isQueryWithDistinct(query.expr) &&
// Queries with $__range variable should not be split because then the interpolated $__range variable is incorrect
// because it is interpolated on the backend with the split timeRange
!isQueryWithRangeVariable(query.expr)

13
public/app/plugins/datasource/loki/queryUtils.test.ts
View File

@ -12,7 +12,6 @@ import {
getParserFromQuery,
obfuscate,
requestSupportsSplitting,
isQueryWithDistinct,
isQueryWithRangeVariable,
isQueryPipelineErrorFiltering,
getLogQueryFromMetricsQuery,
@ -310,18 +309,6 @@ describe('isQueryWithLabelFormat', () => {
});
});
describe('isQueryWithDistinct', () => {
it('identifies queries using distinct', () => {
expect(isQueryWithDistinct('{job="grafana"} | distinct id')).toBe(true);
expect(isQueryWithDistinct('count_over_time({job="grafana"} | distinct id [1m])')).toBe(true);
});
it('does not return false positives', () => {
expect(isQueryWithDistinct('{label="distinct"} | logfmt')).toBe(false);
expect(isQueryWithDistinct('count_over_time({job="distinct"} | json [1m])')).toBe(false);
});
});
describe('isQueryWithRangeVariableDuration', () => {
it('identifies queries using $__range variable', () => {
expect(isQueryWithRangeVariable('rate({job="grafana"}[$__range])')).toBe(true);

5
public/app/plugins/datasource/loki/queryUtils.ts
View File

@ -17,7 +17,6 @@ import {
MetricExpr,
Matcher,
Identifier,
Distinct,
Range,
formatLokiQuery,
} from '@grafana/lezer-logql';
@ -248,10 +247,6 @@ export function isQueryWithLineFilter(query: string): boolean {
return isQueryWithNode(query, LineFilter);
}
export function isQueryWithDistinct(query: string): boolean {
return isQueryWithNode(query, Distinct);
}
export function isQueryWithRangeVariable(query: string): boolean {
const rangeNodes = getNodesFromQuery(query, [Range]);
for (const node of rangeNodes) {

22
public/app/plugins/datasource/loki/querybuilder/operations.ts
View File

@ -1,4 +1,3 @@
import { LabelParamEditor } from '../../prometheus/querybuilder/components/LabelParamEditor';
import {
createAggregationOperation,
createAggregationOperationWithParam,
@ -487,27 +486,6 @@ Example: \`\`error_level=\`level\` \`\`
addOperationHandler: addLokiOperation,
explainHandler: () => `This will remove ANSI color codes from log lines.`,
},
{
id: LokiOperationId.Distinct,
name: 'Distinct',
params: [
{
name: 'Label',
type: 'string',
restParam: true,
optional: true,
editor: LabelParamEditor,
},
],
defaultParams: [''],
alternativesKey: 'format',
category: LokiVisualQueryOperationCategory.Formats,
orderRank: LokiOperationOrder.Unwrap,
renderer: (op, def, innerExpr) => `${innerExpr} | distinct ${op.params.join(',')}`,
addOperationHandler: addLokiOperation,
explainHandler: () =>
'Allows filtering log lines using their original and extracted labels to filter out duplicate label values. The first line occurrence of a distinct value is returned, and the others are dropped.',
},
{
id: LokiOperationId.Drop,
name: 'Drop',

30
public/app/plugins/datasource/loki/querybuilder/parsing.test.ts
View File

@ -747,36 +747,6 @@ describe('buildVisualQueryFromString', () => {
});
});
it('parses a log query with distinct and no labels', () => {
expect(buildVisualQueryFromString('{app="frontend"} | distinct')).toEqual(
noErrors({
labels: [
{
op: '=',
value: 'frontend',
label: 'app',
},
],
operations: [{ id: LokiOperationId.Distinct, params: [] }],
})
);
});
it('parses a log query with distinct and labels', () => {
expect(buildVisualQueryFromString('{app="frontend"} | distinct id, email')).toEqual(
noErrors({
labels: [
{
op: '=',
value: 'frontend',
label: 'app',
},
],
operations: [{ id: LokiOperationId.Distinct, params: ['id', 'email'] }],
})
);
});
it('parses a log query with drop and no labels', () => {
expect(buildVisualQueryFromString('{app="frontend"} | drop')).toEqual(
noErrors({

24
public/app/plugins/datasource/loki/querybuilder/parsing.ts
View File

@ -8,8 +8,6 @@ import {
By,
ConvOp,
Decolorize,
DistinctFilter,
DistinctLabel,
DropLabel,
DropLabels,
DropLabelsExpr,
@ -213,11 +211,6 @@ export function handleExpression(expr: string, node: SyntaxNode, context: Contex
break;
}
case DistinctFilter: {
visQuery.operations.push(handleDistinctFilter(expr, node, context));
break;
}
case DropLabelsExpr: {
visQuery.operations.push(handleDropFilter(expr, node, context));
break;
@ -660,23 +653,6 @@ function isEmptyQuery(query: LokiVisualQuery) {
return false;
}
function handleDistinctFilter(expr: string, node: SyntaxNode, context: Context): QueryBuilderOperation {
const labels: string[] = [];
let exploringNode = node.getChild(DistinctLabel);
while (exploringNode) {
const label = getString(expr, exploringNode.getChild(Identifier));
if (label) {
labels.push(label);
}
exploringNode = exploringNode?.getChild(DistinctLabel);
}
labels.reverse();
return {
id: LokiOperationId.Distinct,
params: labels,
};
}
function handleDropFilter(expr: string, node: SyntaxNode, context: Context): QueryBuilderOperation {
const labels: string[] = [];
let exploringNode = node.getChild(DropLabels);

1
public/app/plugins/datasource/loki/querybuilder/types.ts
View File

@ -38,7 +38,6 @@ export enum LokiOperationId {
Regexp = 'regexp',
Pattern = 'pattern',
Unpack = 'unpack',
Distinct = 'distinct',
LineFormat = 'line_format',
LabelFormat = 'label_format',
Decolorize = 'decolorize',

12
yarn.lock
View File

@ -3867,14 +3867,12 @@ __metadata:
languageName: node
linkType: hard
"@grafana/lezer-logql@npm:0.1.9":
version: 0.1.9
resolution: "@grafana/lezer-logql@npm:0.1.9"
dependencies:
lodash: ^4.17.21
"@grafana/lezer-logql@npm:0.1.11":
version: 0.1.11
resolution: "@grafana/lezer-logql@npm:0.1.11"
peerDependencies:
"@lezer/lr": ^1.0.0
checksum: 6cceb18586413864137ef2305bbe2fdce054796c61a3fde4864e3f2d30ea3dac853aa701728867bf609a4009334dbf92fddbc06f7291dbca9459a86c9dc29e67
checksum: 6a624b9a8d31ff854fcf9708c35e6a7498e78c4bda884639681d0b6d0fffe5527fbaeab1198e5a7694f913181657334345f31156a4a15ff64e3019b30ba6ca2a
languageName: node
linkType: hard
@ -19255,7 +19253,7 @@ __metadata:
"@grafana/faro-core": 1.1.2
"@grafana/faro-web-sdk": 1.1.2
"@grafana/google-sdk": 0.1.1
"@grafana/lezer-logql": 0.1.9
"@grafana/lezer-logql": 0.1.11
"@grafana/lezer-traceql": 0.0.4
"@grafana/monaco-logql": ^0.0.7
"@grafana/runtime": "workspace:*"