InfluxDB: Fix template variable interpolation (#80971)

* use regex as templateSrv replace format

* use regex as templateSrv replace format for raw queries

* import path fix

* don't use regex formatter

* tag value escape

* tag value escape with wrappers

* polished interpolation logic

* update unit tests

* comments and more place to update

* unit test update

* fix escaping

* handle the string and array of string type separately

* update variable type
This commit is contained in:
ismail simsek 2024-02-01 00:30:21 +01:00 committed by GitHub
parent c9bdf69a46
commit 177fa1b947
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 284 additions and 82 deletions

View File

@ -13,8 +13,11 @@ import (
)
var (
regexpOperatorPattern = regexp.MustCompile(`^\/.*\/$`)
regexpMeasurementPattern = regexp.MustCompile(`^\/.*\/$`)
regexpOperatorPattern = regexp.MustCompile(`^\/.*\/$`)
regexpMeasurementPattern = regexp.MustCompile(`^\/.*\/$`)
regexMatcherSimple = regexp.MustCompile(`^/(.*)/$`)
regexMatcherWithStartEndPattern = regexp.MustCompile(`^/\^(.*)\$/$`)
mustEscapeCharsMatcher = regexp.MustCompile(`[\\^$*+?.()|[\]{}\/]`)
)
func (query *Query) Build(queryContext *backend.QueryDataRequest) (string, error) {
@ -103,20 +106,20 @@ func (query *Query) renderTags() []string {
textValue = fmt.Sprintf("'%s'", strings.ReplaceAll(tag.Value, `\`, `\\`))
}
return textValue, operator
return removeRegexWrappers(textValue, `'`), operator
}
// quote value unless regex or number
var textValue string
switch tag.Operator {
case "=~", "!~":
textValue = tag.Value
case "=~", "!~", "":
textValue = escape(tag.Value)
case "<", ">", ">=", "<=":
textValue = tag.Value
textValue = removeRegexWrappers(tag.Value, `'`)
case "Is", "Is Not":
textValue, tag.Operator = isOperatorTypeHandler(tag)
default:
textValue = fmt.Sprintf("'%s'", strings.ReplaceAll(tag.Value, `\`, `\\`))
textValue = fmt.Sprintf("'%s'", strings.ReplaceAll(removeRegexWrappers(tag.Value, ""), `\`, `\\`))
}
escapedKey := fmt.Sprintf(`"%s"`, tag.Key)
@ -244,3 +247,56 @@ func epochMStoInfluxTime(tr *backend.TimeRange) (string, string) {
return fmt.Sprintf("%dms", from), fmt.Sprintf("%dms", to)
}
func removeRegexWrappers(wrappedValue string, wrapper string) string {
value := wrappedValue
// get the value only in between /^...$/
matches := regexMatcherWithStartEndPattern.FindStringSubmatch(wrappedValue)
if len(matches) > 1 {
// full match. the value is like /^value$/
value = wrapper + matches[1] + wrapper
}
return value
}
func escape(unescapedValue string) string {
pipe := `|`
beginning := `/^`
ending := `$/`
value := unescapedValue
substitute := `\$0`
fullMatch := false
// get the value only in between /^...$/
matches := regexMatcherWithStartEndPattern.FindStringSubmatch(unescapedValue)
if len(matches) > 1 {
// full match. the value is like /^value$/
value = matches[1]
fullMatch = true
}
if !fullMatch {
// get the value only in between /.../
matches = regexMatcherSimple.FindStringSubmatch(unescapedValue)
if len(matches) > 1 {
value = matches[1]
beginning = `/`
ending = `/`
}
}
// split them with pipe |
parts := strings.Split(value, pipe)
for i, v := range parts {
// escape each item
parts[i] = mustEscapeCharsMatcher.ReplaceAllString(v, substitute)
}
// stitch them to each other
escaped := make([]byte, 0, 64)
escaped = append(escaped, beginning...)
escaped = append(escaped, strings.Join(parts, pipe)...)
escaped = append(escaped, ending...)
return string(escaped)
}

View File

@ -303,5 +303,53 @@ func TestInfluxdbQueryBuilder(t *testing.T) {
require.Equal(t, query.renderMeasurement(), ` FROM "policy"./apa/`)
})
t.Run("can render regexp tags", func(t *testing.T) {
query := &Query{Tags: []*Tag{{Operator: "=~", Value: `/etc/hosts|/etc/hostname`, Key: "key"}}}
require.Equal(t, `"key" =~ /^\/etc\/hosts|\/etc\/hostname$/`, strings.Join(query.renderTags(), ""))
})
t.Run("can render regexp tags 2", func(t *testing.T) {
query := &Query{Tags: []*Tag{{Operator: "=~", Value: `/^/etc/hosts$/`, Key: "key"}}}
require.Equal(t, `"key" =~ /^\/etc\/hosts$/`, strings.Join(query.renderTags(), ""))
})
t.Run("can render regexp tags 3", func(t *testing.T) {
query := &Query{Tags: []*Tag{{Operator: "=~", Value: `/etc/hosts`, Key: "key"}}}
require.Equal(t, `"key" =~ /^\/etc\/hosts$/`, strings.Join(query.renderTags(), ""))
})
t.Run("can render regexp tags with dots in values", func(t *testing.T) {
query := &Query{Tags: []*Tag{{Operator: "=~", Value: `/etc/resolv.conf`, Key: "key"}}}
require.Equal(t, `"key" =~ /^\/etc\/resolv\.conf$/`, strings.Join(query.renderTags(), ""))
})
t.Run("can render single quoted tag value when regexed value has been sent", func(t *testing.T) {
query := &Query{Tags: []*Tag{{Operator: ">", Value: `/^12.2$/`, Key: "key"}}}
require.Equal(t, `"key" > '12.2'`, strings.Join(query.renderTags(), ""))
})
})
}
func TestRemoveRegexWrappers(t *testing.T) {
t.Run("remove regex wrappers", func(t *testing.T) {
wrappedText := `/^someValue$/`
expected := `'someValue'`
result := removeRegexWrappers(wrappedText, `'`)
require.Equal(t, expected, result)
})
t.Run("return same value if the value is not wrapped by regex wrappers", func(t *testing.T) {
wrappedText := `someValue`
expected := `someValue`
result := removeRegexWrappers(wrappedText, "")
require.Equal(t, expected, result)
})
}

View File

@ -5,9 +5,10 @@ import { BackendSrvRequest } from '@grafana/runtime/';
import config from 'app/core/config';
import { TemplateSrv } from '../../../features/templating/template_srv';
import { queryBuilder } from '../../../features/variables/shared/testing/builders';
import { BROWSER_MODE_DISABLED_MESSAGE } from './constants';
import InfluxDatasource, { influxSpecialRegexEscape } from './datasource';
import InfluxDatasource from './datasource';
import {
getMockDSInstanceSettings,
getMockInfluxDS,
@ -258,6 +259,7 @@ describe('InfluxDataSource Frontend Mode', () => {
const text2 = 'interpolationText2';
const textWithoutFormatRegex = 'interpolationText,interpolationText2';
const textWithFormatRegex = 'interpolationText,interpolationText2';
const justText = 'interpolationText';
const variableMap: Record<string, string> = {
$interpolationVar: text,
$interpolationVar2: text2,
@ -287,14 +289,14 @@ describe('InfluxDataSource Frontend Mode', () => {
function influxChecks(query: InfluxQuery) {
expect(templateSrv.replace).toBeCalledTimes(12);
expect(query.alias).toBe(text);
expect(query.measurement).toBe(textWithFormatRegex);
expect(query.policy).toBe(textWithFormatRegex);
expect(query.limit).toBe(textWithFormatRegex);
expect(query.slimit).toBe(textWithFormatRegex);
expect(query.measurement).toBe(justText);
expect(query.policy).toBe(justText);
expect(query.limit).toBe(justText);
expect(query.slimit).toBe(justText);
expect(query.tz).toBe(text);
expect(query.tags![0].value).toBe(textWithFormatRegex);
expect(query.groupBy![0].params![0]).toBe(textWithFormatRegex);
expect(query.select![0][0].params![0]).toBe(textWithFormatRegex);
expect(query.groupBy![0].params![0]).toBe(justText);
expect(query.select![0][0].params![0]).toBe(justText);
expect(query.adhocFilters?.[0].key).toBe(adhocFilters[0].key);
}
@ -342,7 +344,12 @@ describe('InfluxDataSource Frontend Mode', () => {
});
describe('variable interpolation with chained variables with frontend mode', () => {
const mockTemplateService = new TemplateSrv();
const variablesMock = [queryBuilder().withId('var1').withName('var1').withCurrent('var1').build()];
const mockTemplateService = new TemplateSrv({
getVariables: () => variablesMock,
getVariableWithName: (name: string) => variablesMock.filter((v) => v.name === name)[0],
getFilteredVariables: jest.fn(),
});
mockTemplateService.getAdhocFilters = jest.fn((_: string) => []);
let ds = getMockInfluxDS(getMockDSInstanceSettings(), mockTemplateService);
const fetchMockImpl = () =>
@ -410,18 +417,82 @@ describe('InfluxDataSource Frontend Mode', () => {
});
});
describe('influxSpecialRegexEscape', () => {
it('should escape the dot properly', () => {
const value = 'value.with-dot';
const expectation = `value\.with-dot`;
const result = influxSpecialRegexEscape(value);
describe('interpolateQueryExpr', () => {
let ds = getMockInfluxDS(getMockDSInstanceSettings(), new TemplateSrv());
it('should return the value as it is', () => {
const value = 'normalValue';
const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build();
const result = ds.interpolateQueryExpr(value, variableMock, 'my query $tempVar');
const expectation = 'normalValue';
expect(result).toBe(expectation);
});
it('should escape the url properly', () => {
const value = 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/jolokia';
const expectation = `https:\/\/aaaa-aa-aaa\.bbb\.ccc\.ddd:8443\/jolokia`;
const result = influxSpecialRegexEscape(value);
it('should return the escaped value if the value wrapped in regex', () => {
const value = '/special/path';
const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build();
const result = ds.interpolateQueryExpr(value, variableMock, 'select that where path = /$tempVar/');
const expectation = `\\/special\\/path`;
expect(result).toBe(expectation);
});
it('should return the escaped value if the value wrapped in regex 2', () => {
const value = '/special/path';
const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build();
const result = ds.interpolateQueryExpr(value, variableMock, 'select that where path = /^$tempVar$/');
const expectation = `\\/special\\/path`;
expect(result).toBe(expectation);
});
it('should **not** return the escaped value if the value **is not** wrapped in regex', () => {
const value = '/special/path';
const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build();
const result = ds.interpolateQueryExpr(value, variableMock, `select that where path = '$tempVar'`);
const expectation = `/special/path`;
expect(result).toBe(expectation);
});
it('should **not** return the escaped value if the value **is not** wrapped in regex 2', () => {
const value = '12.2';
const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti(false).build();
const result = ds.interpolateQueryExpr(value, variableMock, `select that where path = '$tempVar'`);
const expectation = `12.2`;
expect(result).toBe(expectation);
});
it('should escape the value **always** if the variable is a multi-value variable', () => {
const value = [`/special/path`, `/some/other/path`];
const variableMock = queryBuilder().withId('tempVar').withName('tempVar').withMulti().build();
const result = ds.interpolateQueryExpr(value, variableMock, `select that where path = '$tempVar'`);
const expectation = `\\/special\\/path|\\/some\\/other\\/path`;
expect(result).toBe(expectation);
});
it('should escape and join with the pipe even the variable is not multi-value', () => {
const variableMock = queryBuilder()
.withId('tempVar')
.withName('tempVar')
.withCurrent('All', '$__all')
.withMulti(false)
.withAllValue('')
.withIncludeAll()
.withOptions(
{
text: 'All',
value: '$__all',
},
{
text: `/special/path`,
value: `/special/path`,
},
{
text: `/some/other/path`,
value: `/some/other/path`,
}
)
.build();
const value = [`/special/path`, `/some/other/path`];
const result = ds.interpolateQueryExpr(value, variableMock, `select that where path = /$tempVar/`);
const expectation = `\\/special\\/path|\\/some\\/other\\/path`;
expect(result).toBe(expectation);
});
});

View File

@ -17,6 +17,7 @@ import {
FieldType,
MetricFindValue,
QueryResultMeta,
QueryVariableModel,
RawTimeRange,
ScopedVars,
TIME_SERIES_TIME_FIELD_NAME,
@ -31,7 +32,6 @@ import {
frameToMetricFindValue,
getBackendSrv,
} from '@grafana/runtime';
import { CustomFormatterVariable } from '@grafana/scenes';
import { QueryFormat, SQLQuery } from '@grafana/sql';
import config from 'app/core/config';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
@ -213,7 +213,12 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
return {
...query,
datasource: this.getRef(),
query: this.templateSrv.replace(query.query ?? '', scopedVars, this.interpolateQueryExpr), // The raw query text
query: this.templateSrv.replace(
query.query ?? '',
scopedVars,
(value: string | string[] = [], variable: QueryVariableModel) =>
this.interpolateQueryExpr(value, variable, query.query)
), // The raw query text
};
}
@ -231,9 +236,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
expandedQuery.groupBy = query.groupBy.map((groupBy) => {
return {
...groupBy,
params: groupBy.params?.map((param) => {
return this.templateSrv.replace(param.toString(), undefined, this.interpolateQueryExpr);
}),
params: groupBy.params?.map((param) => this.templateSrv.replace(param.toString(), undefined)),
};
});
}
@ -243,9 +246,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
return selects.map((select) => {
return {
...select,
params: select.params?.map((param) => {
return this.templateSrv.replace(param.toString(), undefined, this.interpolateQueryExpr);
}),
params: select.params?.map((param) => this.templateSrv.replace(param.toString(), undefined)),
};
});
});
@ -255,8 +256,8 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
expandedQuery.tags = query.tags.map((tag) => {
return {
...tag,
key: this.templateSrv.replace(tag.key, scopedVars, this.interpolateQueryExpr),
value: this.templateSrv.replace(tag.value, scopedVars, this.interpolateQueryExpr),
key: this.templateSrv.replace(tag.key, scopedVars),
value: this.templateSrv.replace(tag.value, scopedVars, 'pipe'),
};
});
}
@ -264,34 +265,61 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
return {
...expandedQuery,
adhocFilters: this.templateSrv.getAdhocFilters(this.name) ?? [],
query: this.templateSrv.replace(query.query ?? '', scopedVars, this.interpolateQueryExpr), // The raw query text
rawSql: this.templateSrv.replace(query.rawSql ?? '', scopedVars, this.interpolateQueryExpr), // The raw query text
query: this.templateSrv.replace(
query.query ?? '',
scopedVars,
(value: string | string[] = [], variable: QueryVariableModel) =>
this.interpolateQueryExpr(value, variable, query.query)
), // The raw sql query text
rawSql: this.templateSrv.replace(
query.rawSql ?? '',
scopedVars,
(value: string | string[] = [], variable: QueryVariableModel) =>
this.interpolateQueryExpr(value, variable, query.rawSql)
), // The raw sql query text
alias: this.templateSrv.replace(query.alias ?? '', scopedVars),
limit: this.templateSrv.replace(query.limit?.toString() ?? '', scopedVars, this.interpolateQueryExpr),
measurement: this.templateSrv.replace(query.measurement ?? '', scopedVars, this.interpolateQueryExpr),
policy: this.templateSrv.replace(query.policy ?? '', scopedVars, this.interpolateQueryExpr),
slimit: this.templateSrv.replace(query.slimit?.toString() ?? '', scopedVars, this.interpolateQueryExpr),
limit: this.templateSrv.replace(query.limit?.toString() ?? '', scopedVars),
measurement: this.templateSrv.replace(query.measurement ?? '', scopedVars),
policy: this.templateSrv.replace(query.policy ?? '', scopedVars),
slimit: this.templateSrv.replace(query.slimit?.toString() ?? '', scopedVars),
tz: this.templateSrv.replace(query.tz ?? '', scopedVars),
};
}
interpolateQueryExpr(value: string | string[] = [], variable: Partial<CustomFormatterVariable>) {
// if no multi or include all do not regexEscape
if (!variable.multi && !variable.includeAll) {
return influxRegularEscape(value);
// this should only be used for rawQueries.
interpolateQueryExpr(value: string | string[] = [], variable: QueryVariableModel, query?: string) {
// If there is no query just return the value directly
if (!query) {
return value;
}
if (typeof value === 'string') {
return influxSpecialRegexEscape(value);
// If template variable is a multi-value variable
// we always want to deal with special chars.
if (variable.multi) {
if (typeof value === 'string') {
return escapeRegex(value);
}
// If the value is a string array first escape them then join them with pipe
return value.map((v) => escapeRegex(v)).join('|');
}
const escapedValues = value.map((val) => influxSpecialRegexEscape(val));
// If the variable is not a multi-value variable
// we want to see how it's been used. If it is used in a regex expression
// we escape it. Otherwise, we return it directly.
// regex below checks if the variable inside /^...$/ (^ and $ is optional)
// i.e. /^$myVar$/ or /$myVar/
const regex = new RegExp(`\\/(?:\\^)?\\$${variable.name}(?:\\$)?\\/`, 'gm');
if (regex.test(query)) {
if (typeof value === 'string') {
return escapeRegex(value);
}
if (escapedValues.length === 1) {
return escapedValues[0];
// If the value is a string array first escape them then join them with pipe
return value.map((v) => escapeRegex(v)).join('|');
}
return escapedValues.join('|');
return value;
}
async runMetadataQuery(target: InfluxQuery): Promise<MetricFindValue[]> {
@ -322,7 +350,11 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
).then(this.toMetricFindValue);
}
const interpolated = this.templateSrv.replace(query, options?.scopedVars, this.interpolateQueryExpr);
const interpolated = this.templateSrv.replace(
query,
options?.scopedVars,
(value: string | string[] = [], variable: QueryVariableModel) => this.interpolateQueryExpr(value, variable, query)
);
return lastValueFrom(this._seriesQuery(interpolated, options)).then((resp) => {
return this.responseParser.parse(query, resp);
@ -668,7 +700,12 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
const target: InfluxQuery = {
refId: 'metricFindQuery',
datasource: this.getRef(),
query: this.templateSrv.replace(annotation.query, undefined, this.interpolateQueryExpr),
query: this.templateSrv.replace(
annotation.query,
undefined,
(value: string | string[] = [], variable: QueryVariableModel) =>
this.interpolateQueryExpr(value, variable, annotation.query)
),
rawQuery: true,
};
@ -696,7 +733,9 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
const timeFilter = this.getTimeFilter({ rangeRaw: options.range.raw, timezone: options.timezone });
let query = annotation.query.replace('$timeFilter', timeFilter);
query = this.templateSrv.replace(query, undefined, this.interpolateQueryExpr);
query = this.templateSrv.replace(query, undefined, (value: string | string[] = [], variable: QueryVariableModel) =>
this.interpolateQueryExpr(value, variable, query)
);
return lastValueFrom(this._seriesQuery(query, options)).then((data) => {
if (!data || !data.results || !data.results[0]) {
@ -780,23 +819,3 @@ function timeSeriesToDataFrame(timeSeries: TimeSeries): DataFrame {
length: values.length,
};
}
export function influxRegularEscape(value: string | string[]) {
if (typeof value === 'string') {
// Check the value is a number. If not run to escape special characters
if (isNaN(parseFloat(value))) {
return escapeRegex(value);
}
}
return value;
}
export function influxSpecialRegexEscape(value: string | string[]) {
if (typeof value !== 'string') {
return value;
}
value = value.replace(/\\/g, '\\\\\\\\');
value = value.replace(/[$^*{}\[\]\'+?.()|]/g, '$&');
return value;
}

View File

@ -5,6 +5,7 @@ import { FetchResponse } from '@grafana/runtime/src';
import config from 'app/core/config';
import { TemplateSrv } from '../../../features/templating/template_srv';
import { queryBuilder } from '../../../features/variables/shared/testing/builders';
import InfluxDatasource from './datasource';
import {
@ -149,6 +150,7 @@ describe('InfluxDataSource Backend Mode', () => {
const text2 = 'interpolationText2';
const textWithoutFormatRegex = 'interpolationText,interpolationText2';
const textWithFormatRegex = 'interpolationText,interpolationText2';
const justText = 'interpolationText';
const variableMap: Record<string, string> = {
$interpolationVar: text,
$interpolationVar2: text2,
@ -178,14 +180,14 @@ describe('InfluxDataSource Backend Mode', () => {
function influxChecks(query: InfluxQuery) {
expect(templateSrv.replace).toBeCalledTimes(12);
expect(query.alias).toBe(text);
expect(query.measurement).toBe(textWithFormatRegex);
expect(query.policy).toBe(textWithFormatRegex);
expect(query.limit).toBe(textWithFormatRegex);
expect(query.slimit).toBe(textWithFormatRegex);
expect(query.measurement).toBe(justText);
expect(query.policy).toBe(justText);
expect(query.limit).toBe(justText);
expect(query.slimit).toBe(justText);
expect(query.tz).toBe(text);
expect(query.tags![0].value).toBe(textWithFormatRegex);
expect(query.groupBy![0].params![0]).toBe(textWithFormatRegex);
expect(query.select![0][0].params![0]).toBe(textWithFormatRegex);
expect(query.groupBy![0].params![0]).toBe(justText);
expect(query.select![0][0].params![0]).toBe(justText);
expect(query.adhocFilters?.[0].key).toBe(adhocFilters[0].key);
}
@ -216,7 +218,12 @@ describe('InfluxDataSource Backend Mode', () => {
});
describe('variable interpolation with chained variables with backend mode', () => {
const mockTemplateService = new TemplateSrv();
const variablesMock = [queryBuilder().withId('var1').withName('var1').withCurrent('var1').build()];
const mockTemplateService = new TemplateSrv({
getVariables: () => variablesMock,
getVariableWithName: (name: string) => variablesMock.filter((v) => v.name === name)[0],
getFilteredVariables: jest.fn(),
});
mockTemplateService.getAdhocFilters = jest.fn((_: string) => []);
let ds = getMockInfluxDS(getMockDSInstanceSettings(), mockTemplateService);
const fetchMockImpl = () =>
@ -263,6 +270,7 @@ describe('InfluxDataSource Backend Mode', () => {
},
});
const qe = `SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`;
expect(fetchMock).toHaveBeenCalled();
const qData = fetchMock.mock.calls[0][0].data.queries[0].query;
expect(qData).toBe(qe);
});

View File

@ -8,14 +8,14 @@ import {
FieldType,
PluginType,
ScopedVars,
} from '@grafana/data/src';
} from '@grafana/data';
import {
BackendDataSourceResponse,
FetchResponse,
getBackendSrv,
setBackendSrv,
VariableInterpolation,
} from '@grafana/runtime/src';
} from '@grafana/runtime';
import { SQLQuery } from '@grafana/sql';
import { TemplateSrv } from '../../../features/templating/template_srv';