From 6fa25df37fc1813fa66119d67e6dfe8a1e688ea2 Mon Sep 17 00:00:00 2001 From: Galen Kistler <109082771+gtk-grafana@users.noreply.github.com> Date: Mon, 22 Jul 2024 07:25:52 -0500 Subject: [PATCH] Loki: Fix ad hoc filters adding stream selectors to stream selectors and line filters (#90626) * fix: fix case where we are adding stream selectors to both stream selectors and line filters --------- Co-authored-by: Sven Grossmann --- .../datasource/loki/datasource.test.ts | 41 +++- .../app/plugins/datasource/loki/datasource.ts | 2 +- .../datasource/loki/modifyQuery.test.ts | 194 +++++++++++++----- .../plugins/datasource/loki/modifyQuery.ts | 23 +++ 4 files changed, 209 insertions(+), 51 deletions(-) diff --git a/public/app/plugins/datasource/loki/datasource.test.ts b/public/app/plugins/datasource/loki/datasource.test.ts index ea0292f3630..e61cb73150b 100644 --- a/public/app/plugins/datasource/loki/datasource.test.ts +++ b/public/app/plugins/datasource/loki/datasource.test.ts @@ -281,16 +281,18 @@ describe('LokiDatasource', () => { }; const query: LokiQuery = { expr: originalQuery, refId: 'A' }; const ds = createLokiDatasource(templateSrv); - const adhocFilters = [ + const adhocFilters: AdHocFilter[] = [ { key: 'k1', operator: '=', value: 'v1', + condition: '', }, { key: 'k2', operator: '!=', value: 'v2', + condition: '', }, ]; jest.spyOn(ds, 'addAdHocFilters'); @@ -303,6 +305,13 @@ describe('LokiDatasource', () => { expect(ds.applyTemplateVariables(query, {}, adhocFilters).expr).toBe( 'rate({bar="baz", job="foo", k1="v1", k2!="v2"} |= "bar" [5m])' ); + + assertAdHocFilters( + originalQuery, + 'rate({bar="baz", job="foo", k1="v1", k2!="v2"} |= "bar" [$__auto])', + ds, + adhocFilters + ); }); }); @@ -1285,6 +1294,36 @@ describe('LokiDatasource', () => { assertAdHocFilters('{job="grafana"}', '{job="grafana", instance=~".*"}', ds, defaultAdHocFilters); }); }); + + describe('bug', () => { + beforeEach(() => { + ds = createLokiDatasource(); + }); + const defaultAdHocFilters: AdHocFilter[] = [ + { + key: 'service_name', + operator: '=', + value: 'grafana/hosted-grafana-gateway', + condition: '', + }, + ]; + it('should not add indexed fields twice as index filter and line filter, backtick', () => { + assertAdHocFilters( + '{service_name=`grafana/hosted-grafana-gateway`} | logfmt', + '{service_name="grafana/hosted-grafana-gateway"} | logfmt', + ds, + defaultAdHocFilters + ); + }); + it('should not add indexed fields twice as index filter and line filter, quotes', () => { + assertAdHocFilters( + '{service_name="grafana/hosted-grafana-gateway"} | logfmt', + '{service_name="grafana/hosted-grafana-gateway"} | logfmt', + ds, + defaultAdHocFilters + ); + }); + }); }); describe('logs volume data provider', () => { diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index e63c317ff30..c4cc01c9357 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -1043,7 +1043,7 @@ export class LokiDatasource * @todo this.templateSrv.getAdhocFilters() is deprecated */ addAdHocFilters(queryExpr: string, adhocFilters?: AdHocVariableFilter[]) { - if (!adhocFilters) { + if (!adhocFilters?.length) { return queryExpr; } diff --git a/public/app/plugins/datasource/loki/modifyQuery.test.ts b/public/app/plugins/datasource/loki/modifyQuery.test.ts index 2a1598dcc20..5ad9b36022f 100644 --- a/public/app/plugins/datasource/loki/modifyQuery.test.ts +++ b/public/app/plugins/datasource/loki/modifyQuery.test.ts @@ -6,6 +6,8 @@ import { addLineFilter, addNoPipelineErrorToQuery, addParserToQuery, + getIdentifierInStreamPositions, + getStreamSelectorPositions, NodePosition, queryHasFilter, removeCommentsFromQuery, @@ -15,55 +17,64 @@ import { LabelType } from './types'; describe('addLabelToQuery()', () => { it.each` - query | description | label | operator | value | expectedResult - ${'{x="y"}'} | ${'no label and value'} | ${''} | ${'='} | ${''} | ${''} - ${'{x="yy"}'} | ${'simple query'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="yy", bar="baz"}'} - ${'{x="yy"}'} | ${'simple query'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="yy", bar="baz"}'} - ${'{x="yy"}'} | ${'custom operator'} | ${'bar'} | ${'!='} | ${'baz'} | ${'{x="yy", bar!="baz"}'} - ${'rate({}[1m])'} | ${'do not modify ranges'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[1m])'} - ${'sum by (host) (rate({} [1m]))'} | ${'detect in-order function use'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by (host) (rate({bar="baz"}[1m]))'} - ${'{instance="my-host.com:9100"}'} | ${'selectors with punctuation'} | ${'bar'} | ${'='} | ${'baz'} | ${'{instance="my-host.com:9100", bar="baz"}'} - ${'{list="a,b,c"}'} | ${'selectors with punctuation'} | ${'bar'} | ${'='} | ${'baz'} | ${'{list="a,b,c", bar="baz"}'} - ${'rate({}[5m]) + rate({}[5m])'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[5m]) + rate({bar="baz"}[5m])'} - ${'avg(rate({x="y"} [$__interval]))+ sum(rate({}[5m]))'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'avg(rate({x="y", bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} - ${'rate({x="yy"}[5m]) * rate({y="zz",a="bb"}[5m]) * rate({}[5m])'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({x="yy", bar="baz"}[5m]) * rate({y="zz", a="bb", bar="baz"}[5m]) * rate({bar="baz"}[5m])'} - ${'{x="yy", bar!="baz"}'} | ${'do not add duplicate labels'} | ${'bar'} | ${'!='} | ${'baz'} | ${'{x="yy", bar!="baz"}'} - ${'rate({bar="baz"}[1m])'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[1m])'} - ${'{list="a,b,c", bar="baz"}'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'{list="a,b,c", bar="baz"}'} - ${'avg(rate({bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'avg(rate({bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} - ${'{x="y"} |="yy"'} | ${'do not remove filters'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} |="yy"'} - ${'{x="y"} |="yy" !~"xx"'} | ${'do not remove filters'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} |="yy" !~"xx"'} - ${'{x="y"} or {}'} | ${'metric with logical operators'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} or {bar="baz"}'} - ${'{x="y"} and {}'} | ${'metric with logical operators'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} and {bar="baz"}'} - ${'sum(rate({job="foo"}[2m])) by (value $variable)'} | ${'template variables'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({job="foo", bar="baz"}[2m])) by (value $variable)'} - ${'rate({x="y"}[${__range_s}s])'} | ${'metric query with range grafana variable'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({x="y", bar="baz"}[${__range_s}s])'} - ${'max by (id, name, type) ({type=~"foo|bar|baz-test"}) * on(id) group_right(id, type, name) sum by (id) (rate({} [5m])) * 1000'} | ${'metric query with labels in label list with the group modifier'} | ${'bar'} | ${'='} | ${'baz'} | ${'max by (id, name, type) ({type=~"foo|bar|baz-test", bar="baz"}) * on(id) group_right(id, type, name) sum by (id) (rate({bar="baz"}[5m])) * 1000'} - ${'{foo="bar"} | logfmt'} | ${'query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz`'} - ${'{foo="bar"} | logfmt | json'} | ${'query with multiple parsers'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | json | bar=`baz`'} - ${'{foo="bar"} | logfmt | x="y"'} | ${'query with parser and label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | x="y" | bar=`baz`'} - ${'rate({foo="bar"} | logfmt [5m])'} | ${'metric query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({foo="bar"} | logfmt | bar=`baz` [5m])'} - ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | line_format "{{.status}}" [5m]))'} | ${'metric query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | bar=`baz` | line_format "{{.status}}" [5m]))'} - ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | label_format process="{{.process}}" [5m]))'} | ${'metric query with parser and label format'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | label_format process="{{.process}}" | bar=`baz` [5m]))'} - ${'{foo="bar"} | logfmt | x="y" | label_format process="{{.process}}"'} | ${'query with parser and label format'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | x="y" | label_format process="{{.process}}" | bar=`baz`'} - ${'{foo="bar"} | logfmt | line_format "{{.status}}"'} | ${'do not add filter to line_format expressions in query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz` | line_format "{{.status}}"'} - ${'{foo="bar"} | logfmt | line_format "{{status}}"'} | ${'do not add filter to line_format expressions in query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz` | line_format "{{status}}"'} - ${'{}'} | ${'query without stream selector'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}'} - ${'{} | logfmt'} | ${'query without stream selector and with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| logfmt'} - ${'{} | x="y"'} | ${'query without stream selector and with label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| x="y"'} - ${'{} | logfmt | x="y"'} | ${'query without stream selector and with parser and label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| logfmt | x="y"'} - ${'sum(rate({x="y"} [5m])) + sum(rate({} | logfmt [5m]))'} | ${'metric query with 1 empty and 1 not empty stream selector with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({x="y", bar="baz"} [5m])) + sum(rate({bar="baz"}| logfmt [5m]))'} - ${'sum(rate({x="y"} | logfmt [5m])) + sum(rate({} [5m]))'} | ${'metric query with 1 non-empty and 1 not empty stream selector with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({x="y", bar="baz"} | logfmt [5m])) + sum(rate({bar="baz"}[5m]))'} - ${'{x="yy"}'} | ${'simple query with escaped value'} | ${'bar'} | ${'='} | ${'"baz"'} | ${'{x="yy", bar=""baz""}'} - ${'{x="yy"}'} | ${'simple query with escaped value'} | ${'bar'} | ${'='} | ${'\\"baz\\"'} | ${'{x="yy", bar="\\"baz\\""}'} - ${'{x="yy"}'} | ${'simple query with an other escaped value'} | ${'bar'} | ${'='} | ${'baz\\\\'} | ${'{x="yy", bar="baz\\\\"}'} - ${'{x="yy"}'} | ${'simple query with escaped value and regex operator'} | ${'bar'} | ${'~='} | ${'baz\\\\'} | ${'{x="yy", bar~="baz\\\\"}'} - ${'{foo="bar"} | logfmt'} | ${'query with parser with escaped value'} | ${'bar'} | ${'='} | ${'\\"baz\\"'} | ${'{foo="bar"} | logfmt | bar=`"baz"`'} - ${'{foo="bar"} | logfmt'} | ${'query with parser with an other escaped value'} | ${'bar'} | ${'='} | ${'baz\\\\'} | ${'{foo="bar"} | logfmt | bar=`baz\\`'} - ${'{foo="bar"} | logfmt'} | ${'query with parser with escaped value and regex operator'} | ${'bar'} | ${'~='} | ${'\\"baz\\"'} | ${'{foo="bar"} | logfmt | bar~=`"baz"`'} - ${'{foo="bar"} | logfmt'} | ${'query with parser with escaped value and regex operator'} | ${'bar'} | ${'~='} | ${'\\"baz\\"'} | ${'{foo="bar"} | logfmt | bar~=`"baz"`'} - ${'{foo="bar"} | logfmt'} | ${'query with parser, > operator and number value'} | ${'bar'} | ${'>'} | ${'5'} | ${'{foo="bar"} | logfmt | bar>5'} - ${'{foo="bar"} | logfmt'} | ${'query with parser, < operator and non-number value'} | ${'bar'} | ${'<'} | ${'5KiB'} | ${'{foo="bar"} | logfmt | bar<`5KiB`'} - ${'sum(rate({x="y"} | logfmt [5m])) + sum(rate({x="z"} | logfmt [5m]))'} | ${'metric query with non empty selectors and parsers'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({x="y"} | logfmt | bar=`baz` [5m])) + sum(rate({x="z"} | logfmt | bar=`baz` [5m]))'} + query | description | label | operator | value | expectedResult + ${'{x="y"}'} | ${'no label and value'} | ${''} | ${'='} | ${''} | ${''} + ${'{x="yy"}'} | ${'simple query'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="yy", bar="baz"}'} + ${'{x="yy"}'} | ${'simple query'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="yy", bar="baz"}'} + ${'{x="yy"}'} | ${'custom operator'} | ${'bar'} | ${'!='} | ${'baz'} | ${'{x="yy", bar!="baz"}'} + ${'rate({}[1m])'} | ${'do not modify ranges'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[1m])'} + ${'sum by (host) (rate({} [1m]))'} | ${'detect in-order function use'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by (host) (rate({bar="baz"}[1m]))'} + ${'{instance="my-host.com:9100"}'} | ${'selectors with punctuation'} | ${'bar'} | ${'='} | ${'baz'} | ${'{instance="my-host.com:9100", bar="baz"}'} + ${'{list="a,b,c"}'} | ${'selectors with punctuation'} | ${'bar'} | ${'='} | ${'baz'} | ${'{list="a,b,c", bar="baz"}'} + ${'rate({}[5m]) + rate({}[5m])'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[5m]) + rate({bar="baz"}[5m])'} + ${'avg(rate({x="y"} [$__interval]))+ sum(rate({}[5m]))'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'avg(rate({x="y", bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} + ${'rate({x="yy"}[5m]) * rate({y="zz",a="bb"}[5m]) * rate({}[5m])'} | ${'arithmetical expressions'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({x="yy", bar="baz"}[5m]) * rate({y="zz", a="bb", bar="baz"}[5m]) * rate({bar="baz"}[5m])'} + ${'{x="yy", bar!="baz"}'} | ${'do not add duplicate labels'} | ${'bar'} | ${'!='} | ${'baz'} | ${'{x="yy", bar!="baz"}'} + ${'rate({bar="baz"}[1m])'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({bar="baz"}[1m])'} + ${'{list="a,b,c", bar="baz"}'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'{list="a,b,c", bar="baz"}'} + ${'avg(rate({bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} | ${'do not add duplicate labels'} | ${'bar'} | ${'='} | ${'baz'} | ${'avg(rate({bar="baz"} [$__interval]))+ sum(rate({bar="baz"}[5m]))'} + ${'{x="y"} |="yy"'} | ${'do not remove filters'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} |="yy"'} + ${'{x="y"} |="yy" !~"xx"'} | ${'do not remove filters'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} |="yy" !~"xx"'} + ${'{x="y"} or {}'} | ${'metric with logical operators'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} or {bar="baz"}'} + ${'{x="y"} and {}'} | ${'metric with logical operators'} | ${'bar'} | ${'='} | ${'baz'} | ${'{x="y", bar="baz"} and {bar="baz"}'} + ${'sum(rate({job="foo"}[2m])) by (value $variable)'} | ${'template variables'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({job="foo", bar="baz"}[2m])) by (value $variable)'} + ${'rate({x="y"}[${__range_s}s])'} | ${'metric query with range grafana variable'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({x="y", bar="baz"}[${__range_s}s])'} + ${'max by (id, name, type) ({type=~"foo|bar|baz-test"}) * on(id) group_right(id, type, name) sum by (id) (rate({} [5m])) * 1000'} | ${'metric query with labels in label list with the group modifier'} | ${'bar'} | ${'='} | ${'baz'} | ${'max by (id, name, type) ({type=~"foo|bar|baz-test", bar="baz"}) * on(id) group_right(id, type, name) sum by (id) (rate({bar="baz"}[5m])) * 1000'} + ${'{foo="bar"} | logfmt'} | ${'query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz`'} + ${'{foo="bar"} | logfmt | json'} | ${'query with multiple parsers'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | json | bar=`baz`'} + ${'{foo="bar"} | logfmt | x="y"'} | ${'query with parser and label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | x="y" | bar=`baz`'} + ${'rate({foo="bar"} | logfmt [5m])'} | ${'metric query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'rate({foo="bar"} | logfmt | bar=`baz` [5m])'} + ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | line_format "{{.status}}" [5m]))'} | ${'metric query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | bar=`baz` | line_format "{{.status}}" [5m]))'} + ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | label_format process="{{.process}}" [5m]))'} | ${'metric query with parser and label format'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum by(host) (rate({foo="bar"} | logfmt | x="y" | label_format process="{{.process}}" | bar=`baz` [5m]))'} + ${'{foo="bar"} | logfmt | x="y" | label_format process="{{.process}}"'} | ${'query with parser and label format'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | x="y" | label_format process="{{.process}}" | bar=`baz`'} + ${'{foo="bar"} | logfmt | line_format "{{.status}}"'} | ${'do not add filter to line_format expressions in query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz` | line_format "{{.status}}"'} + ${'{foo="bar"} | logfmt | line_format "{{status}}"'} | ${'do not add filter to line_format expressions in query with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{foo="bar"} | logfmt | bar=`baz` | line_format "{{status}}"'} + ${'{}'} | ${'query without stream selector'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}'} + ${'{} | logfmt'} | ${'query without stream selector and with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| logfmt'} + ${'{} | x="y"'} | ${'query without stream selector and with label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| x="y"'} + ${'{} | logfmt | x="y"'} | ${'query without stream selector and with parser and label filter'} | ${'bar'} | ${'='} | ${'baz'} | ${'{bar="baz"}| logfmt | x="y"'} + ${'sum(rate({x="y"} [5m])) + sum(rate({} | logfmt [5m]))'} | ${'metric query with 1 empty and 1 not empty stream selector with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({x="y", bar="baz"} [5m])) + sum(rate({bar="baz"}| logfmt [5m]))'} + ${'sum(rate({x="y"} | logfmt [5m])) + sum(rate({} [5m]))'} | ${'metric query with 1 non-empty and 1 not empty stream selector with parser'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({x="y", bar="baz"} | logfmt [5m])) + sum(rate({bar="baz"}[5m]))'} + ${'sum(rate({x="y", bar="baz"} | logfmt [5m])) + sum(rate({x="y", bar="baz"} [5m]))'} | ${'metric query with two duplicate stream selectors'} | ${'x'} | ${'='} | ${'y'} | ${'sum(rate({x="y", bar="baz"} | logfmt [5m])) + sum(rate({x="y", bar="baz"} [5m]))'} + ${'{x="yy"}'} | ${'simple query with escaped value'} | ${'bar'} | ${'='} | ${'"baz"'} | ${'{x="yy", bar=""baz""}'} + ${'{x="yy"}'} | ${'simple query with escaped value'} | ${'bar'} | ${'='} | ${'\\"baz\\"'} | ${'{x="yy", bar="\\"baz\\""}'} + ${'{x="yy"}'} | ${'simple query with an other escaped value'} | ${'bar'} | ${'='} | ${'baz\\\\'} | ${'{x="yy", bar="baz\\\\"}'} + ${'{x="yy"}'} | ${'simple query with escaped value and regex operator'} | ${'bar'} | ${'~='} | ${'baz\\\\'} | ${'{x="yy", bar~="baz\\\\"}'} + ${'{foo="bar"} | logfmt'} | ${'query with parser with escaped value'} | ${'bar'} | ${'='} | ${'\\"baz\\"'} | ${'{foo="bar"} | logfmt | bar=`"baz"`'} + ${'{foo=`"bar"`} | logfmt'} | ${'query with label already added to stream selector, doublequotes/backticks'} | ${'foo'} | ${'='} | ${`"bar"`} | ${'{foo=""bar""} | logfmt'} + ${'{foo="\\"bar\\""`} | logfmt'} | ${'query with label already added to stream selector, doublequotes/escaped'} | ${'foo'} | ${'='} | ${'\\"bar\\"'} | ${'{foo="\\"bar\\""} | logfmt'} + ${'{foo="bar"} | logfmt'} | ${'query with label already added to stream selector, doublequotes unescaped'} | ${'foo'} | ${'='} | ${'bar'} | ${'{foo="bar"} | logfmt'} + ${'{foo=`bar`} | logfmt'} | ${'query with label already added to stream selector, backticks'} | ${'foo'} | ${'='} | ${'bar'} | ${'{foo="bar"} | logfmt'} + ${'{service_name=`grafana/hosted-grafana-gateway`} | logfmt | caller!=`handler.go:637` '} | ${'query with parser and line filter, backticks'} | ${'service_name'} | ${'='} | ${'grafana/hosted-grafana-gateway'} | ${'{service_name="grafana/hosted-grafana-gateway"} | logfmt | caller!=`handler.go:637` '} + ${'{service_name=`grafana/hosted-grafana-gateway`, pod_template_hash!=`5fd76866f4`} | logfmt | caller!=`handler.go:637`'} | ${'query with parser and line filter, multiple stream selectors'} | ${'service_name'} | ${'='} | ${'grafana/hosted-grafana-gateway'} | ${'{service_name="grafana/hosted-grafana-gateway", pod_template_hash!="5fd76866f4"} | logfmt | caller!=`handler.go:637`'} + ${'{service_name=`grafana/hosted-grafana-gateway`, x!=`y`} | logfmt | caller!=`handler.go:637`'} | ${'query with parser and line filter, multiple stream selectors, value2'} | ${'x'} | ${'!='} | ${'y'} | ${'{service_name="grafana/hosted-grafana-gateway", x!="y"} | logfmt | caller!=`handler.go:637`'} + ${'{service_name="grafana/hosted-grafana-gateway"} | logfmt | caller!=`handler.go:637` '} | ${'query with parser and line filter, doublequotes'} | ${'service_name'} | ${'='} | ${'grafana/hosted-grafana-gateway'} | ${'{service_name="grafana/hosted-grafana-gateway"} | logfmt | caller!=`handler.go:637` '} + ${'{foo="bar"} | logfmt'} | ${'query with parser with an other escaped value'} | ${'bar'} | ${'='} | ${'baz\\\\'} | ${'{foo="bar"} | logfmt | bar=`baz\\`'} + ${'{foo="bar"} | logfmt'} | ${'query with parser with escaped value and regex operator'} | ${'bar'} | ${'~='} | ${'\\"baz\\"'} | ${'{foo="bar"} | logfmt | bar~=`"baz"`'} + ${'{foo="bar"} | logfmt'} | ${'query with parser with escaped value and regex operator'} | ${'bar'} | ${'~='} | ${'\\"baz\\"'} | ${'{foo="bar"} | logfmt | bar~=`"baz"`'} + ${'{foo="bar"} | logfmt'} | ${'query with parser, > operator and number value'} | ${'bar'} | ${'>'} | ${'5'} | ${'{foo="bar"} | logfmt | bar>5'} + ${'{foo="bar"} | logfmt'} | ${'query with parser, < operator and non-number value'} | ${'bar'} | ${'<'} | ${'5KiB'} | ${'{foo="bar"} | logfmt | bar<`5KiB`'} + ${'sum(rate({x="y"} | logfmt [5m])) + sum(rate({x="z"} | logfmt [5m]))'} | ${'metric query with non empty selectors and parsers'} | ${'bar'} | ${'='} | ${'baz'} | ${'sum(rate({x="y"} | logfmt | bar=`baz` [5m])) + sum(rate({x="z"} | logfmt | bar=`baz` [5m]))'} `( 'should add label to query: $query, description: $description', ({ query, description, label, operator, value, expectedResult }) => { @@ -353,3 +364,88 @@ describe.each(['|=', '!='])('addLineFilter type %s', (op: string) => { ); }); }); + +describe('getStreamSelectorPositions', () => { + it('should parse position of stream selectors', () => { + expect( + getStreamSelectorPositions('sum(rate({x="y", bar="baz"} | logfmt [5m])) + sum(rate({x="y", bar="baz"} [5m]))') + ).toEqual([ + { + from: 9, + to: 27, + type: { + name: 'Selector', + props: {}, + id: 40, + flags: 0, + }, + }, + { + from: 55, + to: 73, + type: { + name: 'Selector', + props: {}, + id: 40, + flags: 0, + }, + }, + ]); + }); +}); +describe('getIdentifierInStreamPositions', () => { + it('should parse position of stream selectors', () => { + const indexedKeys = ['x', 'bar']; + const expr = `sum(rate({${indexedKeys[0]}="y", ${indexedKeys[1]}="baz"} | logfmt | x |= "x=y" |= "bar=baz" [5m])) + sum(rate({${indexedKeys[0]}="y", ${indexedKeys[1]}="baz"} [5m]))`; + const identifiers = getIdentifierInStreamPositions(expr); + identifiers.forEach((identifier, index) => { + expect(identifier.getExpression(expr)).toEqual(indexedKeys[index % 2]); + }); + expect(identifiers).toEqual([ + //x1 + { + from: 10, + to: 11, + type: { + name: 'Identifier', + props: {}, + id: 43, + flags: 0, + }, + }, + //bar1 + { + from: 17, + to: 20, + type: { + name: 'Identifier', + props: {}, + id: 43, + flags: 0, + }, + }, + //x2 + { + from: 82, + to: 83, + type: { + name: 'Identifier', + props: {}, + id: 43, + flags: 0, + }, + }, + //bar2 + { + from: 89, + to: 92, + type: { + name: 'Identifier', + props: {}, + id: 43, + flags: 0, + }, + }, + ]); + }); +}); diff --git a/public/app/plugins/datasource/loki/modifyQuery.ts b/public/app/plugins/datasource/loki/modifyQuery.ts index 2f8e2c8c72b..293d78fc660 100644 --- a/public/app/plugins/datasource/loki/modifyQuery.ts +++ b/public/app/plugins/datasource/loki/modifyQuery.ts @@ -165,6 +165,16 @@ export function addLabelToQuery( const hasStreamSelectorMatchers = getMatcherInStreamPositions(query); // For non-indexed labels we want to add them after label_format to, for example, allow ad-hoc filters to use formatted labels const labelFormatPositions = getNodePositionsFromQuery(query, [LabelFormatExpr]); + + // If the label type wasn't passed in from the calling function, we can use lezer to figure out if this label is already in the stream selectors + if (!labelType) { + const identifierSelectorMatchers = getIdentifierInStreamPositions(query); + const indexedKeys = identifierSelectorMatchers.map((match) => match.getExpression(query)); + if (indexedKeys.includes(key)) { + labelType = LabelType.Indexed; + } + } + const everyStreamSelectorHasMatcher = streamSelectorPositions.every((streamSelectorPosition) => hasStreamSelectorMatchers.some( (matcherPosition) => @@ -621,3 +631,16 @@ function getMatcherInStreamPositions(query: string): NodePosition[] { }); return positions; } + +export function getIdentifierInStreamPositions(query: string): NodePosition[] { + const tree = parser.parse(query); + const positions: NodePosition[] = []; + tree.iterate({ + enter: ({ node }): false | void => { + if (node.type.id === Selector) { + positions.push(...getAllPositionsInNodeByType(node, Identifier)); + } + }, + }); + return positions; +}