From 6c7b17b59fa77d8413b90f6395b057e1f591009c Mon Sep 17 00:00:00 2001 From: ismail simsek Date: Sat, 3 Jun 2023 13:39:01 +0300 Subject: [PATCH] InfluxDB: Strong types and backend migration preparation (#69469) * Type definitions * Move non-backend-migration code into unified place * Update public/app/plugins/datasource/influxdb/datasource.ts --- .betterer.results | 19 +- .../plugins/datasource/influxdb/datasource.ts | 513 +++++++++--------- 2 files changed, 264 insertions(+), 268 deletions(-) diff --git a/.betterer.results b/.betterer.results index 6b120495549..b1d0b44e1e1 100644 --- a/.betterer.results +++ b/.betterer.results @@ -4153,39 +4153,34 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "0"] ], "public/app/plugins/datasource/influxdb/datasource.ts:5381": [ - [0, 0, 0, "Do not use any type assertions.", "0"], + [0, 0, 0, "Unexpected any. Specify a different type.", "0"], [0, 0, 0, "Unexpected any. Specify a different type.", "1"], [0, 0, 0, "Unexpected any. Specify a different type.", "2"], [0, 0, 0, "Unexpected any. Specify a different type.", "3"], [0, 0, 0, "Unexpected any. Specify a different type.", "4"], - [0, 0, 0, "Do not use any type assertions.", "5"], - [0, 0, 0, "Unexpected any. Specify a different type.", "6"], + [0, 0, 0, "Unexpected any. Specify a different type.", "5"], + [0, 0, 0, "Do not use any type assertions.", "6"], [0, 0, 0, "Unexpected any. Specify a different type.", "7"], [0, 0, 0, "Unexpected any. Specify a different type.", "8"], [0, 0, 0, "Unexpected any. Specify a different type.", "9"], [0, 0, 0, "Unexpected any. Specify a different type.", "10"], - [0, 0, 0, "Unexpected any. Specify a different type.", "11"], + [0, 0, 0, "Do not use any type assertions.", "11"], [0, 0, 0, "Unexpected any. Specify a different type.", "12"], [0, 0, 0, "Unexpected any. Specify a different type.", "13"], [0, 0, 0, "Unexpected any. Specify a different type.", "14"], [0, 0, 0, "Unexpected any. Specify a different type.", "15"], - [0, 0, 0, "Do not use any type assertions.", "16"], + [0, 0, 0, "Unexpected any. Specify a different type.", "16"], [0, 0, 0, "Unexpected any. Specify a different type.", "17"], [0, 0, 0, "Unexpected any. Specify a different type.", "18"], [0, 0, 0, "Unexpected any. Specify a different type.", "19"], [0, 0, 0, "Unexpected any. Specify a different type.", "20"], - [0, 0, 0, "Do not use any type assertions.", "21"], + [0, 0, 0, "Unexpected any. Specify a different type.", "21"], [0, 0, 0, "Unexpected any. Specify a different type.", "22"], [0, 0, 0, "Unexpected any. Specify a different type.", "23"], [0, 0, 0, "Unexpected any. Specify a different type.", "24"], [0, 0, 0, "Unexpected any. Specify a different type.", "25"], [0, 0, 0, "Unexpected any. Specify a different type.", "26"], - [0, 0, 0, "Unexpected any. Specify a different type.", "27"], - [0, 0, 0, "Unexpected any. Specify a different type.", "28"], - [0, 0, 0, "Unexpected any. Specify a different type.", "29"], - [0, 0, 0, "Unexpected any. Specify a different type.", "30"], - [0, 0, 0, "Unexpected any. Specify a different type.", "31"], - [0, 0, 0, "Unexpected any. Specify a different type.", "32"] + [0, 0, 0, "Do not use any type assertions.", "27"] ], "public/app/plugins/datasource/influxdb/influx_query_model.ts:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], diff --git a/public/app/plugins/datasource/influxdb/datasource.ts b/public/app/plugins/datasource/influxdb/datasource.ts index adb47dd8648..e5f1f3a1e66 100644 --- a/public/app/plugins/datasource/influxdb/datasource.ts +++ b/public/app/plugins/datasource/influxdb/datasource.ts @@ -41,88 +41,16 @@ import { InfluxQueryBuilder } from './query_builder'; import ResponseParser from './response_parser'; import { InfluxOptions, InfluxQuery, InfluxVersion } from './types'; -// we detect the field type based on the value-array -function getFieldType(values: unknown[]): FieldType { - // the values-array may contain a lot of nulls. - // we need the first not-null item - const firstNotNull = values.find((v) => v !== null); - - if (firstNotNull === undefined) { - // we could not find any not-null values - return FieldType.number; - } - - const valueType = typeof firstNotNull; - - switch (valueType) { - case 'string': - return FieldType.string; - case 'boolean': - return FieldType.boolean; - case 'number': - return FieldType.number; - default: - // this should never happen, influxql values - // can only be numbers, strings and booleans. - throw new Error(`InfluxQL: invalid value type ${valueType}`); - } -} - -// this conversion function is specialized to work with the timeseries -// data returned by InfluxDatasource.getTimeSeries() -function timeSeriesToDataFrame(timeSeries: TimeSeries): DataFrame { - const times: number[] = []; - const values: unknown[] = []; - - // the data we process here is not correctly typed. - // the typescript types say every data-point is number|null, - // but in fact it can be string or boolean too. - - const points = timeSeries.datapoints; - for (const point of points) { - values.push(point[0]); - times.push(point[1] as number); - } - - const timeField = { - name: TIME_SERIES_TIME_FIELD_NAME, - type: FieldType.time, - config: {}, - values: times, - }; - - const valueField = { - name: TIME_SERIES_VALUE_FIELD_NAME, - type: getFieldType(values), - config: { - displayNameFromDS: timeSeries.title, - }, - values: values, - labels: timeSeries.tags, - }; - - const fields = [timeField, valueField]; - - return { - name: timeSeries.target, - refId: timeSeries.refId, - meta: timeSeries.meta, - fields, - length: values.length, - }; -} - export default class InfluxDatasource extends DataSourceWithBackend { type: string; urls: string[]; username: string; password: string; name: string; - database: any; - basicAuth: any; - withCredentials: any; + database?: string; + basicAuth?: string; + withCredentials?: boolean; access: 'direct' | 'proxy'; - interval: any; responseParser: ResponseParser; httpMode: string; isFlux: boolean; @@ -146,7 +74,7 @@ export default class InfluxDatasource extends DataSourceWithBackend { - // migrate annotations - if (options.targets.some((target: InfluxQuery) => target.fromAnnotations)) { - const streams: Array> = []; - - for (const target of options.targets) { - if (target.query) { - streams.push( - new Observable((subscriber) => { - this.annotationEvents(options, target) - .then((events) => subscriber.next({ data: [toDataFrame(events)] })) - .catch((ex) => subscriber.error(new Error(ex))) - .finally(() => subscriber.complete()); - }) - ); - } - } - - return merge(...streams); - } - - let timeFilter = this.getTimeFilter(options); - const scopedVars = options.scopedVars; - const targets = cloneDeep(options.targets); - const queryTargets: any[] = []; - - let i, y; - - let allQueries = _map(targets, (target) => { - if (target.hide) { - return ''; - } - - queryTargets.push(target); - - // backward compatibility - scopedVars.interval = scopedVars.__interval; - - return new InfluxQueryModel(target, this.templateSrv, scopedVars).render(true); - }).reduce((acc, current) => { - if (current !== '') { - acc += ';' + current; - } - return acc; - }); - - if (allQueries === '') { - return of({ data: [] }); - } - - // add global adhoc filters to timeFilter - const adhocFilters = this.templateSrv.getAdhocFilters(this.name); - const adhocFiltersFromDashboard = options.targets.flatMap((target: InfluxQuery) => target.adhocFilters ?? []); - if (adhocFilters?.length || adhocFiltersFromDashboard?.length) { - const ahFilters = adhocFilters?.length ? adhocFilters : adhocFiltersFromDashboard; - const tmpQuery = new InfluxQueryModel({ refId: 'A' }, this.templateSrv, scopedVars); - timeFilter += ' AND ' + tmpQuery.renderAdhocFilters(ahFilters); - } - // replace grafana variables - scopedVars.timeFilter = { value: timeFilter }; - - // replace templated variables - allQueries = this.templateSrv.replace(allQueries, scopedVars); - - return this._seriesQuery(allQueries, options).pipe( - map((data: any) => { - if (!data || !data.results) { - return { data: [] }; - } - - const seriesList = []; - for (i = 0; i < data.results.length; i++) { - const result = data.results[i]; - if (!result || !result.series) { - continue; - } - - const target = queryTargets[i]; - let alias = target.alias; - if (alias) { - alias = this.templateSrv.replace(target.alias, options.scopedVars); - } - - const meta: QueryResultMeta = { - executedQueryString: data.executedQueryString, - }; - - const influxSeries = new InfluxSeries({ - refId: target.refId, - series: data.results[i].series, - alias: alias, - meta, - }); - - switch (target.resultFormat) { - case 'logs': - meta.preferredVisualisationType = 'logs'; - case 'table': { - seriesList.push(influxSeries.getTable()); - break; - } - default: { - const timeSeries = influxSeries.getTimeSeries(); - for (y = 0; y < timeSeries.length; y++) { - seriesList.push(timeSeriesToDataFrame(timeSeries[y])); - } - break; - } - } - } - - return { data: seriesList }; - }) - ); - } - - async annotationEvents(options: DataQueryRequest, annotation: InfluxQuery): Promise { - if (this.isFlux) { - return Promise.reject({ - message: 'Flux requires the standard annotation query', - }); - } - - // InfluxQL puts a query string on the annotation - if (!annotation.query) { - return Promise.reject({ - message: 'Query missing in annotation definition', - }); - } - - if (config.featureToggles.influxdbBackendMigration && this.access === 'proxy') { - // We want to send our query to the backend as a raw query - const target: InfluxQuery = { - refId: 'metricFindQuery', - datasource: this.getRef(), - query: this.templateSrv.replace(annotation.query, undefined, 'regex'), - rawQuery: true, - }; - - return lastValueFrom( - getBackendSrv() - .fetch({ - url: '/api/ds/query', - method: 'POST', - headers: this.getRequestHeaders(), - data: { - from: options.range.from.valueOf().toString(), - to: options.range.to.valueOf().toString(), - queries: [target], - }, - requestId: annotation.name, - }) - .pipe( - map( - async (res: FetchResponse) => - await this.responseParser.transformAnnotationResponse(annotation, res, target) - ) - ) - ); - } - - const timeFilter = this.getTimeFilter({ rangeRaw: options.range.raw, timezone: options.timezone }); - let query = annotation.query.replace('$timeFilter', timeFilter); - query = this.templateSrv.replace(query, undefined, 'regex'); - - return lastValueFrom(this._seriesQuery(query, options)).then((data: any) => { - if (!data || !data.results || !data.results[0]) { - throw { message: 'No results in response from InfluxDB' }; - } - return new InfluxSeries({ - series: data.results[0].series, - annotation: annotation, - }).getAnnotations(); - }); - } - targetContainsTemplate(target: any) { // for flux-mode we just take target.query, // for influxql-mode we use InfluxQueryModel to create the text-representation @@ -776,7 +525,259 @@ export default class InfluxDatasource extends DataSourceWithBackend { + // migrate annotations + if (options.targets.some((target: InfluxQuery) => target.fromAnnotations)) { + const streams: Array> = []; + + for (const target of options.targets) { + if (target.query) { + streams.push( + new Observable((subscriber) => { + this.annotationEvents(options, target) + .then((events) => subscriber.next({ data: [toDataFrame(events)] })) + .catch((ex) => subscriber.error(new Error(ex))) + .finally(() => subscriber.complete()); + }) + ); + } + } + + return merge(...streams); + } + + let timeFilter = this.getTimeFilter(options); + const scopedVars = options.scopedVars; + const targets = cloneDeep(options.targets); + const queryTargets: any[] = []; + + let i, y; + + let allQueries = _map(targets, (target) => { + if (target.hide) { + return ''; + } + + queryTargets.push(target); + + // backward compatibility + scopedVars.interval = scopedVars.__interval; + + return new InfluxQueryModel(target, this.templateSrv, scopedVars).render(true); + }).reduce((acc, current) => { + if (current !== '') { + acc += ';' + current; + } + return acc; + }); + + if (allQueries === '') { + return of({ data: [] }); + } + + // add global adhoc filters to timeFilter + const adhocFilters = this.templateSrv.getAdhocFilters(this.name); + const adhocFiltersFromDashboard = options.targets.flatMap((target: InfluxQuery) => target.adhocFilters ?? []); + if (adhocFilters?.length || adhocFiltersFromDashboard?.length) { + const ahFilters = adhocFilters?.length ? adhocFilters : adhocFiltersFromDashboard; + const tmpQuery = new InfluxQueryModel({ refId: 'A' }, this.templateSrv, scopedVars); + timeFilter += ' AND ' + tmpQuery.renderAdhocFilters(ahFilters); + } + // replace grafana variables + scopedVars.timeFilter = { value: timeFilter }; + + // replace templated variables + allQueries = this.templateSrv.replace(allQueries, scopedVars); + + return this._seriesQuery(allQueries, options).pipe( + map((data: any) => { + if (!data || !data.results) { + return { data: [] }; + } + + const seriesList = []; + for (i = 0; i < data.results.length; i++) { + const result = data.results[i]; + if (!result || !result.series) { + continue; + } + + const target = queryTargets[i]; + let alias = target.alias; + if (alias) { + alias = this.templateSrv.replace(target.alias, options.scopedVars); + } + + const meta: QueryResultMeta = { + executedQueryString: data.executedQueryString, + }; + + const influxSeries = new InfluxSeries({ + refId: target.refId, + series: data.results[i].series, + alias: alias, + meta, + }); + + switch (target.resultFormat) { + case 'logs': + meta.preferredVisualisationType = 'logs'; + case 'table': { + seriesList.push(influxSeries.getTable()); + break; + } + default: { + const timeSeries = influxSeries.getTimeSeries(); + for (y = 0; y < timeSeries.length; y++) { + seriesList.push(timeSeriesToDataFrame(timeSeries[y])); + } + break; + } + } + } + + return { data: seriesList }; + }) + ); + } + + async annotationEvents(options: DataQueryRequest, annotation: InfluxQuery): Promise { + if (this.isFlux) { + return Promise.reject({ + message: 'Flux requires the standard annotation query', + }); + } + + // InfluxQL puts a query string on the annotation + if (!annotation.query) { + return Promise.reject({ + message: 'Query missing in annotation definition', + }); + } + + if (this.isMigrationToggleOnAndIsAccessProxy()) { + // We want to send our query to the backend as a raw query + const target: InfluxQuery = { + refId: 'metricFindQuery', + datasource: this.getRef(), + query: this.templateSrv.replace(annotation.query, undefined, 'regex'), + rawQuery: true, + }; + + return lastValueFrom( + getBackendSrv() + .fetch({ + url: '/api/ds/query', + method: 'POST', + headers: this.getRequestHeaders(), + data: { + from: options.range.from.valueOf().toString(), + to: options.range.to.valueOf().toString(), + queries: [target], + }, + requestId: annotation.name, + }) + .pipe( + map( + async (res: FetchResponse) => + await this.responseParser.transformAnnotationResponse(annotation, res, target) + ) + ) + ); + } + + const timeFilter = this.getTimeFilter({ rangeRaw: options.range.raw, timezone: options.timezone }); + let query = annotation.query.replace('$timeFilter', timeFilter); + query = this.templateSrv.replace(query, undefined, 'regex'); + + return lastValueFrom(this._seriesQuery(query, options)).then((data: any) => { + if (!data || !data.results || !data.results[0]) { + throw { message: 'No results in response from InfluxDB' }; + } + return new InfluxSeries({ + series: data.results[0].series, + annotation: annotation, + }).getAnnotations(); + }); + } +} + +// we detect the field type based on the value-array +function getFieldType(values: unknown[]): FieldType { + // the values-array may contain a lot of nulls. + // we need the first not-null item + const firstNotNull = values.find((v) => v !== null); + + if (firstNotNull === undefined) { + // we could not find any not-null values + return FieldType.number; + } + + const valueType = typeof firstNotNull; + + switch (valueType) { + case 'string': + return FieldType.string; + case 'boolean': + return FieldType.boolean; + case 'number': + return FieldType.number; + default: + // this should never happen, influxql values + // can only be numbers, strings and booleans. + throw new Error(`InfluxQL: invalid value type ${valueType}`); + } +} + +// this conversion function is specialized to work with the timeseries +// data returned by InfluxDatasource.getTimeSeries() +function timeSeriesToDataFrame(timeSeries: TimeSeries): DataFrame { + const times: number[] = []; + const values: unknown[] = []; + + // the data we process here is not correctly typed. + // the typescript types say every data-point is number|null, + // but in fact it can be string or boolean too. + + const points = timeSeries.datapoints; + for (const point of points) { + values.push(point[0]); + times.push(point[1] as number); + } + + const timeField = { + name: TIME_SERIES_TIME_FIELD_NAME, + type: FieldType.time, + config: {}, + values: times, + }; + + const valueField = { + name: TIME_SERIES_VALUE_FIELD_NAME, + type: getFieldType(values), + config: { + displayNameFromDS: timeSeries.title, + }, + values: values, + labels: timeSeries.tags, + }; + + const fields = [timeField, valueField]; + + return { + name: timeSeries.target, + refId: timeSeries.refId, + meta: timeSeries.meta, + fields, + length: values.length, + }; }