From a12788a4e707d34e57613d99402cfcbc4a8ec962 Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Wed, 4 Oct 2023 17:02:46 +0200 Subject: [PATCH] InfluxDB: Interpolate variables based on their type (#75653) * Rename the mock function * Move tests * Refactor existing tests * add influxql_metadata_query tests * move to root * remove unnecessary file * adhoc test * Remove unused parameter * tests for future * fix mocks * betterer * interpolate variables based on their types * prettier --- .../datasource/influxdb/datasource.test.ts | 84 +++++++--- .../plugins/datasource/influxdb/datasource.ts | 68 +++++--- .../influxdb/datasource_backend_mode.test.ts | 146 +++++++++++++++++- 3 files changed, 257 insertions(+), 41 deletions(-) diff --git a/public/app/plugins/datasource/influxdb/datasource.test.ts b/public/app/plugins/datasource/influxdb/datasource.test.ts index eeb2ebfbe19..7f5831a6ad0 100644 --- a/public/app/plugins/datasource/influxdb/datasource.test.ts +++ b/public/app/plugins/datasource/influxdb/datasource.test.ts @@ -206,7 +206,7 @@ describe('InfluxDataSource Frontend Mode', () => { const text = 'interpolationText'; const text2 = 'interpolationText2'; const textWithoutFormatRegex = 'interpolationText,interpolationText2'; - const textWithFormatRegex = 'interpolationText|interpolationText2'; + const textWithFormatRegex = 'interpolationText,interpolationText2'; const variableMap: Record = { $interpolationVar: text, $interpolationVar2: text2, @@ -288,30 +288,74 @@ describe('InfluxDataSource Frontend Mode', () => { expect(templateSrv.replace).toBeCalledTimes(1); expect(query.query).toBe(text); }); + }); - it('should apply all template variables with InfluxQL mode', () => { - ds.version = ds.version = InfluxVersion.InfluxQL; - ds.access = 'proxy'; - config.featureToggles.influxdbBackendMigration = true; - const query = ds.applyTemplateVariables(mockInfluxQueryWithTemplateVars(adhocFilters), { - interpolationVar: { text: text, value: text }, - interpolationVar2: { text: 'interpolationText2', value: 'interpolationText2' }, + describe('variable interpolation with chained variables with frontend mode', () => { + const mockTemplateService = new TemplateSrv(); + mockTemplateService.getAdhocFilters = jest.fn((_: string) => []); + let ds = getMockInfluxDS(getMockDSInstanceSettings(), mockTemplateService); + const fetchMockImpl = () => + of({ + data: { + status: 'success', + results: [ + { + series: [ + { + name: 'measurement', + columns: ['name'], + values: [['cpu']], + }, + ], + }, + ], + }, }); - influxChecks(query); + + beforeEach(() => { + jest.clearAllMocks(); + fetchMock.mockImplementation(fetchMockImpl); }); - it('should apply all scopedVars to tags', () => { - ds.version = InfluxVersion.InfluxQL; - ds.access = 'proxy'; - config.featureToggles.influxdbBackendMigration = true; - const query = ds.applyTemplateVariables(mockInfluxQueryWithTemplateVars(adhocFilters), { - interpolationVar: { text: text, value: text }, - interpolationVar2: { text: 'interpolationText2', value: 'interpolationText2' }, + it('should render chained regex variables with floating point number', () => { + ds.metricFindQuery(`SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= $maxSED`, { + scopedVars: { maxSED: { text: '8.1', value: '8.1' } }, }); - expect(query.tags?.length).toBeGreaterThan(0); - const value = query.tags?.[0].value; - const scopedVars = 'interpolationText|interpolationText2'; - expect(value).toBe(scopedVars); + const qe = `SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= 8.1`; + const qData = decodeURIComponent(fetchMock.mock.calls[0][0].data.substring(2)); + expect(qData).toBe(qe); + }); + + it('should render chained regex variables with URL', () => { + ds.metricFindQuery('SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^$var1$/', { + scopedVars: { + var1: { + text: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg', + value: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg', + }, + }, + }); + const qe = `SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`; + const qData = decodeURIComponent(fetchMock.mock.calls[0][0].data.substring(2)); + expect(qData).toBe(qe); + }); + + it('should render chained regex variables with floating point number and url', () => { + ds.metricFindQuery( + 'SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= $maxSED AND agent_url =~ /^$var1$/', + { + scopedVars: { + var1: { + text: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg', + value: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg', + }, + maxSED: { text: '8.1', value: '8.1' }, + }, + } + ); + const qe = `SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= 8.1 AND agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`; + const qData = decodeURIComponent(fetchMock.mock.calls[0][0].data.substring(2)); + expect(qData).toBe(qe); }); }); }); diff --git a/public/app/plugins/datasource/influxdb/datasource.ts b/public/app/plugins/datasource/influxdb/datasource.ts index f19108fcbe4..ba6a4f2a4d7 100644 --- a/public/app/plugins/datasource/influxdb/datasource.ts +++ b/public/app/plugins/datasource/influxdb/datasource.ts @@ -13,6 +13,7 @@ import { DataSourceInstanceSettings, dateMath, DateTime, + escapeRegex, FieldType, MetricFindValue, QueryResultMeta, @@ -30,6 +31,7 @@ import { frameToMetricFindValue, getBackendSrv, } from '@grafana/runtime'; +import { CustomFormatterVariable } from '@grafana/scenes'; import config from 'app/core/config'; import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv'; @@ -251,7 +253,7 @@ export default class InfluxDatasource extends DataSourceWithBackend { - return this.templateSrv.replace(param.toString(), undefined, 'regex'); + return this.templateSrv.replace(param.toString(), undefined, this.interpolateQueryExpr); }), }; }); @@ -282,7 +284,7 @@ export default class InfluxDatasource extends DataSourceWithBackend { - return this.templateSrv.replace(param.toString(), undefined, 'regex'); + return this.templateSrv.replace(param.toString(), undefined, this.interpolateQueryExpr); }), }; }); @@ -293,7 +295,7 @@ export default class InfluxDatasource extends DataSourceWithBackend { return { ...tag, - value: this.templateSrv.replace(tag.value, scopedVars, 'regex'), + value: this.templateSrv.replace(tag.value, scopedVars, this.interpolateQueryExpr), }; }); } @@ -301,16 +303,35 @@ export default class InfluxDatasource extends DataSourceWithBackend) { + // if no multi or include all do not regexEscape + if (!variable.multi && !variable.includeAll) { + return influxRegularEscape(value); + } + + if (typeof value === 'string') { + return influxSpecialRegexEscape(value); + } + + const escapedValues = value.map((val) => influxSpecialRegexEscape(val)); + + if (escapedValues.length === 1) { + return escapedValues[0]; + } + + return escapedValues.join('|'); + } + async runMetadataQuery(target: InfluxQuery): Promise { return lastValueFrom( super.query({ @@ -344,15 +365,7 @@ export default class InfluxDatasource extends DataSourceWithBackend { return this.responseParser.parse(query, resp); @@ -690,7 +703,7 @@ export default class InfluxDatasource extends DataSourceWithBackend { if (!data || !data.results || !data.results[0]) { @@ -802,3 +815,18 @@ function timeSeriesToDataFrame(timeSeries: TimeSeries): DataFrame { length: values.length, }; } + +export function influxRegularEscape(value: string | string[]) { + if (typeof value === 'string') { + // Check the value is a number. If not run to escape special characters + if (isNaN(parseFloat(value))) { + return escapeRegex(value); + } + } + + return value; +} + +export function influxSpecialRegexEscape(value: string | string[]) { + return typeof value === 'string' ? value.replace(/\\/g, '\\\\\\\\').replace(/[$^*{}\[\]\'+?.()|]/g, '\\\\$&') : value; +} diff --git a/public/app/plugins/datasource/influxdb/datasource_backend_mode.test.ts b/public/app/plugins/datasource/influxdb/datasource_backend_mode.test.ts index f5855674c88..c1e5d5f0b64 100644 --- a/public/app/plugins/datasource/influxdb/datasource_backend_mode.test.ts +++ b/public/app/plugins/datasource/influxdb/datasource_backend_mode.test.ts @@ -4,14 +4,18 @@ import { DataQueryRequest, dateTime, ScopedVars } from '@grafana/data/src'; import { FetchResponse } from '@grafana/runtime/src'; import config from 'app/core/config'; +import { TemplateSrv } from '../../../features/templating/template_srv'; + +import InfluxDatasource from './datasource'; import { getMockDSInstanceSettings, getMockInfluxDS, mockBackendService, mockInfluxFetchResponse, + mockInfluxQueryWithTemplateVars, mockTemplateSrv, } from './mocks'; -import { InfluxQuery } from './types'; +import { InfluxQuery, InfluxVersion } from './types'; config.featureToggles.influxdbBackendMigration = true; const fetchMock = mockBackendService(mockInfluxFetchResponse()); @@ -139,4 +143,144 @@ describe('InfluxDataSource Backend Mode', () => { expect(fetchReq.queries[0].tags?.[1].value).toBe(adhocFilters[0].value); }); }); + + describe('when interpolating template variables', () => { + const text = 'interpolationText'; + const text2 = 'interpolationText2'; + const textWithoutFormatRegex = 'interpolationText,interpolationText2'; + const textWithFormatRegex = 'interpolationText,interpolationText2'; + const variableMap: Record = { + $interpolationVar: text, + $interpolationVar2: text2, + }; + const adhocFilters = [ + { + key: 'adhoc', + operator: '=', + value: 'val', + condition: '', + }, + ]; + const templateSrv = mockTemplateSrv( + jest.fn((_: string) => adhocFilters), + jest.fn((target?: string, scopedVars?: ScopedVars, format?: string | Function): string => { + if (!format) { + return variableMap[target!] || ''; + } + if (format === 'regex') { + return textWithFormatRegex; + } + return textWithoutFormatRegex; + }) + ); + const ds = new InfluxDatasource(getMockDSInstanceSettings(), templateSrv); + + function influxChecks(query: InfluxQuery) { + expect(templateSrv.replace).toBeCalledTimes(10); + expect(query.alias).toBe(text); + expect(query.measurement).toBe(textWithFormatRegex); + expect(query.policy).toBe(textWithFormatRegex); + expect(query.limit).toBe(textWithFormatRegex); + expect(query.slimit).toBe(textWithFormatRegex); + expect(query.tz).toBe(text); + expect(query.tags![0].value).toBe(textWithFormatRegex); + expect(query.groupBy![0].params![0]).toBe(textWithFormatRegex); + expect(query.select![0][0].params![0]).toBe(textWithFormatRegex); + expect(query.adhocFilters?.[0].key).toBe(adhocFilters[0].key); + } + + it('should apply all template variables with InfluxQL mode', () => { + ds.version = ds.version = InfluxVersion.InfluxQL; + ds.access = 'proxy'; + const query = ds.applyTemplateVariables(mockInfluxQueryWithTemplateVars(adhocFilters), { + interpolationVar: { text: text, value: text }, + interpolationVar2: { text: 'interpolationText2', value: 'interpolationText2' }, + }); + influxChecks(query); + }); + + it('should apply all scopedVars to tags', () => { + ds.version = InfluxVersion.InfluxQL; + ds.access = 'proxy'; + const query = ds.applyTemplateVariables(mockInfluxQueryWithTemplateVars(adhocFilters), { + interpolationVar: { text: text, value: text }, + interpolationVar2: { text: 'interpolationText2', value: 'interpolationText2' }, + }); + if (!query.tags?.length) { + throw new Error('Tags are not defined'); + } + const value = query.tags[0].value; + const scopedVars = 'interpolationText,interpolationText2'; + expect(value).toBe(scopedVars); + }); + }); + + describe('variable interpolation with chained variables with backend mode', () => { + const mockTemplateService = new TemplateSrv(); + mockTemplateService.getAdhocFilters = jest.fn((_: string) => []); + let ds = getMockInfluxDS(getMockDSInstanceSettings(), mockTemplateService); + const fetchMockImpl = () => + of({ + data: { + status: 'success', + results: [ + { + series: [ + { + name: 'measurement', + columns: ['name'], + values: [['cpu']], + }, + ], + }, + ], + }, + }); + + beforeEach(() => { + jest.clearAllMocks(); + fetchMock.mockImplementation(fetchMockImpl); + }); + + it('should render chained regex variables with floating point number', () => { + ds.metricFindQuery(`SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= $maxSED`, { + scopedVars: { maxSED: { text: '8.1', value: '8.1' } }, + }); + const qe = `SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= 8.1`; + const qData = fetchMock.mock.calls[0][0].data.queries[0].query; + expect(qData).toBe(qe); + }); + + it('should render chained regex variables with URL', () => { + ds.metricFindQuery('SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^$var1$/', { + scopedVars: { + var1: { + text: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg', + value: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg', + }, + }, + }); + const qe = `SHOW TAG VALUES WITH KEY = "agent_url" WHERE agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`; + const qData = fetchMock.mock.calls[0][0].data.queries[0].query; + expect(qData).toBe(qe); + }); + + it('should render chained regex variables with floating point number and url', () => { + ds.metricFindQuery( + 'SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= $maxSED AND agent_url =~ /^$var1$/', + { + scopedVars: { + var1: { + text: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg', + value: 'https://aaaa-aa-aaa.bbb.ccc.ddd:8443/ggggg', + }, + maxSED: { text: '8.1', value: '8.1' }, + }, + } + ); + const qe = `SELECT sum("piece_count") FROM "rp"."pdata" WHERE diameter <= 8.1 AND agent_url =~ /^https:\\/\\/aaaa-aa-aaa\\.bbb\\.ccc\\.ddd:8443\\/ggggg$/`; + const qData = fetchMock.mock.calls[0][0].data.queries[0].query; + expect(qData).toBe(qe); + }); + }); });