IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

InfluxDB: Interpolate variables based on their type (#75653)

Browse Source 
* Rename the mock function

* Move tests

* Refactor existing tests

* add influxql_metadata_query tests

* move to root

* remove unnecessary file

* adhoc test

* Remove unused parameter

* tests for future

* fix mocks

* betterer

* interpolate variables based on their types

* prettier
This commit is contained in:
ismail simsek 2023-10-04 17:02:46 +02:00 committed by GitHub
parent 2d603bed22
commit a12788a4e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 257 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
public/app/plugins/datasource/influxdb/datasource.test.ts
View File

@ -206,7 +206,7 @@ describe('InfluxDataSource Frontend Mode', () => {
const text = 'interpolationText';
const text2 = 'interpolationText2';
const textWithoutFormatRegex = 'interpolationText,interpolationText2';
const textWithFormatRegex = 'interpolationText|interpolationText2';
const textWithFormatRegex = 'interpolationText,interpolationText2';
const variableMap: Record<string, string> = {
$interpolationVar: text,
$interpolationVar2: text2,
@ -288,30 +288,74 @@ describe('InfluxDataSource Frontend Mode', () => {
expect(templateSrv.replace).toBeCalledTimes(1);
expect(query.query).toBe(text);
});
});
it('should apply all template variables with InfluxQL mode', () => {
ds.version = ds.version = InfluxVersion.InfluxQL;
ds.access = 'proxy';
config.featureToggles.influxdbBackendMigration = true;
const query = ds.applyTemplateVariables(mockInfluxQueryWithTemplateVars(adhocFilters), {
interpolationVar: { text: text, value: text },
interpolationVar2: { text: 'interpolationText2', value: 'interpolationText2' },
describe('variable interpolation with chained variables with frontend mode', () => {
const mockTemplateService = new TemplateSrv();
mockTemplateService.getAdhocFilters = jest.fn((_: string) => []);
let ds = getMockInfluxDS(getMockDSInstanceSettings(), mockTemplateService);
const fetchMockImpl = () =>
of({
data: {
status: 'success',
results: [
{
series: [
{
name: 'measurement',
columns: ['name'],
values: [['cpu']],
},
],
},
],
},
});
influxChecks(query);
beforeEach(() => {
jest.clearAllMocks();
fetchMock.mockImplementation(fetchMockImpl);
});
it('should apply all scopedVars to tags', () => {
ds.version = InfluxVersion.InfluxQL;
ds.access = 'proxy';
config.featureToggles.influxdbBackendMigration = true;
const query = ds.applyTemplateVariables(mockInfluxQueryWithTemplateVars(adhocFilters), {
interpolationVar: { text: text, value: text },
interpolationVar2: { text: 'interpolationText2', value: 'interpolationText2' },
it('should render chained regex variables with floating point number', () => {
ds.metricFindQuery(`SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= $maxSED`, {
scopedVars: { maxSED: { text: '8.1', value: '8.1' } },
});
expect(query.tags?.length).toBeGreaterThan(0);
const value = query.tags?.[0].value;
const scopedVars = 'interpolationText|interpolationText2';
expect(value).toBe(scopedVars);
const qe = `SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= 8.1`;
const qData = decodeURIComponent(fetchMock.mock.calls[0][0].data.substring(2));
expect(qData).toBe(qe);
});
it('should render chained regex variables with URL', () => {
ds.metricFindQuery('SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^$var1$/', {
scopedVars: {
var1: {
text: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg',
value: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg',
},
},
});
const qe = `SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`;
const qData = decodeURIComponent(fetchMock.mock.calls[0][0].data.substring(2));
expect(qData).toBe(qe);
});
it('should render chained regex variables with floating point number and url', () => {
ds.metricFindQuery(
'SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= $maxSED AND agent_url =~ /^$var1$/',
{
scopedVars: {
var1: {
text: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg',
value: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg',
},
maxSED: { text: '8.1', value: '8.1' },
},
}
);
const qe = `SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= 8.1 AND agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`;
const qData = decodeURIComponent(fetchMock.mock.calls[0][0].data.substring(2));
expect(qData).toBe(qe);
});
});
});

68
public/app/plugins/datasource/influxdb/datasource.ts
View File

@ -13,6 +13,7 @@ import {
DataSourceInstanceSettings,
dateMath,
DateTime,
escapeRegex,
FieldType,
MetricFindValue,
QueryResultMeta,
@ -30,6 +31,7 @@ import {
frameToMetricFindValue,
getBackendSrv,
} from '@grafana/runtime';
import { CustomFormatterVariable } from '@grafana/scenes';
import config from 'app/core/config';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
@ -251,7 +253,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
return {
...query,
datasource: this.getRef(),
query: this.templateSrv.replace(query.query ?? '', scopedVars, 'regex'), // The raw query text
query: this.templateSrv.replace(query.query ?? '', scopedVars, this.interpolateQueryExpr), // The raw query text
};
}
@ -270,7 +272,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
return {
...groupBy,
params: groupBy.params?.map((param) => {
return this.templateSrv.replace(param.toString(), undefined, 'regex');
return this.templateSrv.replace(param.toString(), undefined, this.interpolateQueryExpr);
}),
};
});
@ -282,7 +284,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
return {
...select,
params: select.params?.map((param) => {
return this.templateSrv.replace(param.toString(), undefined, 'regex');
return this.templateSrv.replace(param.toString(), undefined, this.interpolateQueryExpr);
}),
};
});
@ -293,7 +295,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
expandedQuery.tags = query.tags.map((tag) => {
return {
...tag,
value: this.templateSrv.replace(tag.value, scopedVars, 'regex'),
value: this.templateSrv.replace(tag.value, scopedVars, this.interpolateQueryExpr),
};
});
}
@ -301,16 +303,35 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
return {
...expandedQuery,
adhocFilters: this.templateSrv.getAdhocFilters(this.name) ?? [],
query: this.templateSrv.replace(query.query ?? '', scopedVars, 'regex'), // The raw query text
query: this.templateSrv.replace(query.query ?? '', scopedVars, this.interpolateQueryExpr), // The raw query text
alias: this.templateSrv.replace(query.alias ?? '', scopedVars),
limit: this.templateSrv.replace(query.limit?.toString() ?? '', scopedVars, 'regex'),
measurement: this.templateSrv.replace(query.measurement ?? '', scopedVars, 'regex'),
policy: this.templateSrv.replace(query.policy ?? '', scopedVars, 'regex'),
slimit: this.templateSrv.replace(query.slimit?.toString() ?? '', scopedVars, 'regex'),
limit: this.templateSrv.replace(query.limit?.toString() ?? '', scopedVars, this.interpolateQueryExpr),
measurement: this.templateSrv.replace(query.measurement ?? '', scopedVars, this.interpolateQueryExpr),
policy: this.templateSrv.replace(query.policy ?? '', scopedVars, this.interpolateQueryExpr),
slimit: this.templateSrv.replace(query.slimit?.toString() ?? '', scopedVars, this.interpolateQueryExpr),
tz: this.templateSrv.replace(query.tz ?? '', scopedVars),
};
}
interpolateQueryExpr(value: string | string[] = [], variable: Partial<CustomFormatterVariable>) {
// if no multi or include all do not regexEscape
if (!variable.multi && !variable.includeAll) {
return influxRegularEscape(value);
}
if (typeof value === 'string') {
return influxSpecialRegexEscape(value);
}
const escapedValues = value.map((val) => influxSpecialRegexEscape(val));
if (escapedValues.length === 1) {
return escapedValues[0];
}
return escapedValues.join('|');
}
async runMetadataQuery(target: InfluxQuery): Promise<MetricFindValue[]> {
return lastValueFrom(
super.query({
@ -344,15 +365,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
});
}
const interpolated = new InfluxQueryModel(
{
refId: 'metricFindQuery',
query,
rawQuery: true,
},
this.templateSrv,
options?.scopedVars
).render(true);
const interpolated = this.templateSrv.replace(query, options.scopedVars, this.interpolateQueryExpr);
return lastValueFrom(this._seriesQuery(interpolated, options)).then((resp) => {
return this.responseParser.parse(query, resp);
@ -690,7 +703,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
const target: InfluxQuery = {
refId: 'metricFindQuery',
datasource: this.getRef(),
query: this.templateSrv.replace(annotation.query, undefined, 'regex'),
query: this.templateSrv.replace(annotation.query, undefined, this.interpolateQueryExpr),
rawQuery: true,
};
@ -718,7 +731,7 @@ export default class InfluxDatasource extends DataSourceWithBackend<InfluxQuery,
const timeFilter = this.getTimeFilter({ rangeRaw: options.range.raw, timezone: options.timezone });
let query = annotation.query.replace('$timeFilter', timeFilter);
query = this.templateSrv.replace(query, undefined, 'regex');
query = this.templateSrv.replace(query, undefined, this.interpolateQueryExpr);
return lastValueFrom(this._seriesQuery(query, options)).then((data: any) => {
if (!data || !data.results || !data.results[0]) {
@ -802,3 +815,18 @@ function timeSeriesToDataFrame(timeSeries: TimeSeries): DataFrame {
length: values.length,
};
}
export function influxRegularEscape(value: string | string[]) {
if (typeof value === 'string') {
// Check the value is a number. If not run to escape special characters
if (isNaN(parseFloat(value))) {
return escapeRegex(value);
}
}
return value;
}
export function influxSpecialRegexEscape(value: string | string[]) {
return typeof value === 'string' ? value.replace(/\\/g, '\\\\\\\\').replace(/[$^*{}\[\]\'+?.()|]/g, '\\\\$&') : value;
}

146
public/app/plugins/datasource/influxdb/datasource_backend_mode.test.ts
View File

@ -4,14 +4,18 @@ import { DataQueryRequest, dateTime, ScopedVars } from '@grafana/data/src';
import { FetchResponse } from '@grafana/runtime/src';
import config from 'app/core/config';
import { TemplateSrv } from '../../../features/templating/template_srv';
import InfluxDatasource from './datasource';
import {
getMockDSInstanceSettings,
getMockInfluxDS,
mockBackendService,
mockInfluxFetchResponse,
mockInfluxQueryWithTemplateVars,
mockTemplateSrv,
} from './mocks';
import { InfluxQuery } from './types';
import { InfluxQuery, InfluxVersion } from './types';
config.featureToggles.influxdbBackendMigration = true;
const fetchMock = mockBackendService(mockInfluxFetchResponse());
@ -139,4 +143,144 @@ describe('InfluxDataSource Backend Mode', () => {
expect(fetchReq.queries[0].tags?.[1].value).toBe(adhocFilters[0].value);
});
});
describe('when interpolating template variables', () => {
const text = 'interpolationText';
const text2 = 'interpolationText2';
const textWithoutFormatRegex = 'interpolationText,interpolationText2';
const textWithFormatRegex = 'interpolationText,interpolationText2';
const variableMap: Record<string, string> = {
$interpolationVar: text,
$interpolationVar2: text2,
};
const adhocFilters = [
{
key: 'adhoc',
operator: '=',
value: 'val',
condition: '',
},
];
const templateSrv = mockTemplateSrv(
jest.fn((_: string) => adhocFilters),
jest.fn((target?: string, scopedVars?: ScopedVars, format?: string | Function): string => {
if (!format) {
return variableMap[target!] || '';
}
if (format === 'regex') {
return textWithFormatRegex;
}
return textWithoutFormatRegex;
})
);
const ds = new InfluxDatasource(getMockDSInstanceSettings(), templateSrv);
function influxChecks(query: InfluxQuery) {
expect(templateSrv.replace).toBeCalledTimes(10);
expect(query.alias).toBe(text);
expect(query.measurement).toBe(textWithFormatRegex);
expect(query.policy).toBe(textWithFormatRegex);
expect(query.limit).toBe(textWithFormatRegex);
expect(query.slimit).toBe(textWithFormatRegex);
expect(query.tz).toBe(text);
expect(query.tags![0].value).toBe(textWithFormatRegex);
expect(query.groupBy![0].params![0]).toBe(textWithFormatRegex);
expect(query.select![0][0].params![0]).toBe(textWithFormatRegex);
expect(query.adhocFilters?.[0].key).toBe(adhocFilters[0].key);
}
it('should apply all template variables with InfluxQL mode', () => {
ds.version = ds.version = InfluxVersion.InfluxQL;
ds.access = 'proxy';
const query = ds.applyTemplateVariables(mockInfluxQueryWithTemplateVars(adhocFilters), {
interpolationVar: { text: text, value: text },
interpolationVar2: { text: 'interpolationText2', value: 'interpolationText2' },
});
influxChecks(query);
});
it('should apply all scopedVars to tags', () => {
ds.version = InfluxVersion.InfluxQL;
ds.access = 'proxy';
const query = ds.applyTemplateVariables(mockInfluxQueryWithTemplateVars(adhocFilters), {
interpolationVar: { text: text, value: text },
interpolationVar2: { text: 'interpolationText2', value: 'interpolationText2' },
});
if (!query.tags?.length) {
throw new Error('Tags are not defined');
}
const value = query.tags[0].value;
const scopedVars = 'interpolationText,interpolationText2';
expect(value).toBe(scopedVars);
});
});
describe('variable interpolation with chained variables with backend mode', () => {
const mockTemplateService = new TemplateSrv();
mockTemplateService.getAdhocFilters = jest.fn((_: string) => []);
let ds = getMockInfluxDS(getMockDSInstanceSettings(), mockTemplateService);
const fetchMockImpl = () =>
of({
data: {
status: 'success',
results: [
{
series: [
{
name: 'measurement',
columns: ['name'],
values: [['cpu']],
},
],
},
],
},
});
beforeEach(() => {
jest.clearAllMocks();
fetchMock.mockImplementation(fetchMockImpl);
});
it('should render chained regex variables with floating point number', () => {
ds.metricFindQuery(`SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= $maxSED`, {
scopedVars: { maxSED: { text: '8.1', value: '8.1' } },
});
const qe = `SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= 8.1`;
const qData = fetchMock.mock.calls[0][0].data.queries[0].query;
expect(qData).toBe(qe);
});
it('should render chained regex variables with URL', () => {
ds.metricFindQuery('SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^$var1$/', {
scopedVars: {
var1: {
text: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg',
value: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg',
},
},
});
const qe = `SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`;
const qData = fetchMock.mock.calls[0][0].data.queries[0].query;
expect(qData).toBe(qe);
});
it('should render chained regex variables with floating point number and url', () => {
ds.metricFindQuery(
'SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= $maxSED AND agent_url =~ /^$var1$/',
{
scopedVars: {
var1: {
text: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg',
value: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg',
},
maxSED: { text: '8.1', value: '8.1' },
},
}
);
const qe = `SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= 8.1 AND agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`;
const qData = fetchMock.mock.calls[0][0].data.queries[0].query;
expect(qData).toBe(qe);
});
});
});