From 81756cd702b2e79aa18ee6c9de67efae1fd702ac Mon Sep 17 00:00:00 2001 From: Ivana Huckova <30407135+ivanahuckova@users.noreply.github.com> Date: Fri, 17 Sep 2021 13:39:26 +0200 Subject: [PATCH] Prometheus: Run Explore range queries trough backend (#39133) * Run Explore range queries trough backend * Remove trailing comma * Add timeRange step alignment to backend * Remove creation of instant query on backend as it is not supported ATM * Remove non-related frontend changes * Pass offset to calculate aligned range trough prom query * Update order in query error message * tableRefIds shouldn't contain undefined refIds * Remove cloning of dataframes when processing * Don't mutate response * Remove ordering of processed frames * Remove df because not needed --- pkg/tsdb/prometheus/prometheus.go | 27 +- pkg/tsdb/prometheus/prometheus_test.go | 39 + pkg/tsdb/prometheus/types.go | 9 + .../datasource/prometheus/datasource.ts | 88 +- .../prometheus/result_transformer.test.ts | 1228 ++++++++++------- .../prometheus/result_transformer.ts | 82 ++ .../plugins/datasource/prometheus/types.ts | 2 + 7 files changed, 923 insertions(+), 552 deletions(-) diff --git a/pkg/tsdb/prometheus/prometheus.go b/pkg/tsdb/prometheus/prometheus.go index 786c0a81631..65737de5170 100644 --- a/pkg/tsdb/prometheus/prometheus.go +++ b/pkg/tsdb/prometheus/prometheus.go @@ -51,6 +51,7 @@ type QueryModel struct { RangeQuery bool `json:"range"` InstantQuery bool `json:"instant"` IntervalFactor int64 `json:"intervalFactor"` + OffsetSec int64 `json:"offsetSec"` } type Service struct { @@ -162,13 +163,20 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest) span.SetTag("stop_unixnano", query.End.UnixNano()) defer span.Finish() - value, _, err := client.QueryRange(ctx, query.Expr, timeRange) + var results model.Value - if err != nil { - return &result, err + switch query.QueryType { + case Range: + results, _, err = client.QueryRange(ctx, query.Expr, timeRange) + if err != nil { + return &result, fmt.Errorf("query: %s failed with: %v", query.Expr, err) + } + + default: + return &result, fmt.Errorf("unknown Query type detected %#v", query.QueryType) } - frame, err := parseResponse(value, query) + frame, err := parseResponse(results, query) if err != nil { return &result, err } @@ -283,13 +291,20 @@ func (s *Service) parseQuery(queryContext *backend.QueryDataRequest, dsInfo *Dat expr = strings.ReplaceAll(expr, "$__range", strconv.FormatInt(rangeS, 10)+"s") expr = strings.ReplaceAll(expr, "$__rate_interval", intervalv2.FormatDuration(calculateRateInterval(interval, dsInfo.TimeInterval, s.intervalCalculator))) + queryType := Range + + // Align query range to step. It rounds start and end down to a multiple of step. + start := int64(math.Floor((float64(query.TimeRange.From.Unix()+model.OffsetSec)/interval.Seconds()))*interval.Seconds() - float64(model.OffsetSec)) + end := int64(math.Floor((float64(query.TimeRange.To.Unix()+model.OffsetSec)/interval.Seconds()))*interval.Seconds() - float64(model.OffsetSec)) + qs = append(qs, &PrometheusQuery{ Expr: expr, Step: interval, LegendFormat: model.LegendFormat, - Start: query.TimeRange.From, - End: query.TimeRange.To, + Start: time.Unix(start, 0), + End: time.Unix(end, 0), RefId: query.RefID, + QueryType: queryType, }) } diff --git a/pkg/tsdb/prometheus/prometheus_test.go b/pkg/tsdb/prometheus/prometheus_test.go index 3293c167252..d29a53bf63d 100644 --- a/pkg/tsdb/prometheus/prometheus_test.go +++ b/pkg/tsdb/prometheus/prometheus_test.go @@ -275,6 +275,45 @@ func TestPrometheus_parseQuery(t *testing.T) { require.NoError(t, err) require.Equal(t, "rate(ALERTS{job=\"test\" [1m]})", models[0].Expr) }) + + t.Run("parsing query model of range query", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ + "expr": "go_goroutines", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "range": true + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, Range, models[0].QueryType) + }) + + t.Run("parsing query model of with default query type", func(t *testing.T) { + timeRange := backend.TimeRange{ + From: now, + To: now.Add(48 * time.Hour), + } + + query := queryContext(`{ + "expr": "go_goroutines", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + }`, timeRange) + + dsInfo := &DatasourceInfo{} + models, err := service.parseQuery(query, dsInfo) + require.NoError(t, err) + require.Equal(t, Range, models[0].QueryType) + }) } func queryContext(json string, timeRange backend.TimeRange) *backend.QueryDataRequest { diff --git a/pkg/tsdb/prometheus/types.go b/pkg/tsdb/prometheus/types.go index cf8c16682e8..6016cc25a7f 100644 --- a/pkg/tsdb/prometheus/types.go +++ b/pkg/tsdb/prometheus/types.go @@ -9,4 +9,13 @@ type PrometheusQuery struct { Start time.Time End time.Time RefId string + QueryType PrometheusQueryType } + +type PrometheusQueryType string + +const ( + Range PrometheusQueryType = "range" + //This is currently not used, but we will use it in next iteration + Instant PrometheusQueryType = "instant" +) diff --git a/public/app/plugins/datasource/prometheus/datasource.ts b/public/app/plugins/datasource/prometheus/datasource.ts index a58fb999373..7af13716379 100644 --- a/public/app/plugins/datasource/prometheus/datasource.ts +++ b/public/app/plugins/datasource/prometheus/datasource.ts @@ -8,7 +8,6 @@ import { DataQueryError, DataQueryRequest, DataQueryResponse, - DataSourceApi, DataSourceInstanceSettings, dateMath, DateTime, @@ -17,7 +16,7 @@ import { ScopedVars, TimeRange, } from '@grafana/data'; -import { BackendSrvRequest, FetchError, FetchResponse, getBackendSrv } from '@grafana/runtime'; +import { BackendSrvRequest, FetchError, FetchResponse, getBackendSrv, DataSourceWithBackend } from '@grafana/runtime'; import { safeStringifyValue } from 'app/core/utils/explore'; import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv'; @@ -26,7 +25,7 @@ import addLabelToQuery from './add_label_to_query'; import PrometheusLanguageProvider from './language_provider'; import { expandRecordingRules } from './language_utils'; import { getInitHints, getQueryHints } from './query_hints'; -import { getOriginalMetricName, renderTemplate, transform } from './result_transformer'; +import { getOriginalMetricName, renderTemplate, transform, transformV2 } from './result_transformer'; import { ExemplarTraceIdDestination, isFetchErrorResponse, @@ -47,12 +46,13 @@ export const ANNOTATION_QUERY_STEP_DEFAULT = '60s'; const EXEMPLARS_NOT_AVAILABLE = 'Exemplars for this query are not available.'; const GET_AND_POST_METADATA_ENDPOINTS = ['api/v1/query', 'api/v1/query_range', 'api/v1/series', 'api/v1/labels']; -export class PrometheusDatasource extends DataSourceApi { +export class PrometheusDatasource extends DataSourceWithBackend { type: string; editorSrc: string; ruleMappings: { [index: string]: string }; url: string; directUrl: string; + access: 'direct' | 'proxy'; basicAuth: any; withCredentials: any; metricsNameCache = new LRU(10); @@ -75,6 +75,7 @@ export class PrometheusDatasource extends DataSourceApi this.type = 'prometheus'; this.editorSrc = 'app/features/prometheus/partials/query.editor.html'; this.url = instanceSettings.url!; + this.access = instanceSettings.access; this.basicAuth = instanceSettings.basicAuth; this.withCredentials = instanceSettings.withCredentials; this.interval = instanceSettings.jsonData.timeInterval || '15s'; @@ -288,24 +289,52 @@ export class PrometheusDatasource extends DataSourceApi }; }; + prepareOptionsV2 = (options: DataQueryRequest) => { + const targets = options.targets.map((target) => { + // We want to format Explore + range queries as time_series + return { + ...target, + instant: false, + range: true, + format: 'time_series', + offsetSec: this.timeSrv.timeRange().to.utcOffset() * 60, + }; + }); + + return { ...options, targets }; + }; + query(options: DataQueryRequest): Observable { - const start = this.getPrometheusTime(options.range.from, false); - const end = this.getPrometheusTime(options.range.to, true); - const { queries, activeTargets } = this.prepareTargets(options, start, end); + // WIP - currently we want to run trough backend only if all queries are explore + range queries + const shouldRunBackendQuery = + this.access === 'proxy' && + options.app === CoreApp.Explore && + !options.targets.some((query) => query.exemplar) && + !options.targets.some((query) => query.instant); - // No valid targets, return the empty result to save a round trip. - if (!queries || !queries.length) { - return of({ - data: [], - state: LoadingState.Done, - }); + if (shouldRunBackendQuery) { + const newOptions = this.prepareOptionsV2(options); + return super.query(newOptions).pipe(map((response) => transformV2(response, newOptions))); + // Run queries trough browser/proxy + } else { + const start = this.getPrometheusTime(options.range.from, false); + const end = this.getPrometheusTime(options.range.to, true); + const { queries, activeTargets } = this.prepareTargets(options, start, end); + + // No valid targets, return the empty result to save a round trip. + if (!queries || !queries.length) { + return of({ + data: [], + state: LoadingState.Done, + }); + } + + if (options.app === CoreApp.Explore) { + return this.exploreQuery(queries, activeTargets, end); + } + + return this.panelsQuery(queries, activeTargets, end, options.requestId, options.scopedVars); } - - if (options.app === CoreApp.Explore) { - return this.exploreQuery(queries, activeTargets, end); - } - - return this.panelsQuery(queries, activeTargets, end, options.requestId, options.scopedVars); } private exploreQuery(queries: PromQueryRequest[], activeTargets: PromQuery[], end: number) { @@ -824,6 +853,27 @@ export class PrometheusDatasource extends DataSourceApi getOriginalMetricName(labelData: { [key: string]: string }) { return getOriginalMetricName(labelData); } + + // Used when running queries trough backend + filterQuery(query: PromQuery): boolean { + if (query.hide || !query.expr) { + return false; + } + return true; + } + + // Used when running queries trough backend + applyTemplateVariables(target: PromQuery, scopedVars: ScopedVars): Record { + const variables = cloneDeep(scopedVars); + // We want to interpolate these variables on backend + delete variables.__interval; + delete variables.__interval_ms; + + return { + ...target, + expr: this.templateSrv.replace(target.expr, variables, this.interpolateQueryExpr), + }; + } } /** diff --git a/public/app/plugins/datasource/prometheus/result_transformer.test.ts b/public/app/plugins/datasource/prometheus/result_transformer.test.ts index 1e47b246fc0..f34358d29d0 100644 --- a/public/app/plugins/datasource/prometheus/result_transformer.test.ts +++ b/public/app/plugins/datasource/prometheus/result_transformer.test.ts @@ -1,5 +1,6 @@ -import { DataFrame, FieldType } from '@grafana/data'; -import { transform } from './result_transformer'; +import { DataFrame, FieldType, DataQueryRequest, DataQueryResponse, MutableDataFrame } from '@grafana/data'; +import { transform, transformV2, transformDFoTable } from './result_transformer'; +import { PromQuery } from './types'; jest.mock('@grafana/runtime', () => ({ getTemplateSrv: () => ({ @@ -31,556 +32,729 @@ const matrixResponse = { }; describe('Prometheus Result Transformer', () => { - const options: any = { target: {}, query: {} }; - describe('When nothing is returned', () => { - it('should return empty array', () => { - const response = { - status: 'success', - data: { - resultType: '', - result: null, - }, - }; - const series = transform({ data: response } as any, options); - expect(series).toEqual([]); - }); - it('should return empty array', () => { - const response = { - status: 'success', - data: { - resultType: '', - result: null, - }, - }; - const result = transform({ data: response } as any, { ...options, target: { format: 'table' } }); - expect(result).toHaveLength(0); - }); - }); - - describe('When resultFormat is table', () => { - const response = { - status: 'success', - data: { - resultType: 'matrix', - result: [ + describe('transformV2', () => { + it('results with time_series format should be enriched with preferredVisualisationType', () => { + const options = ({ + targets: [ { - metric: { __name__: 'test', job: 'testjob' }, - values: [ - [1443454528, '3846'], - [1443454530, '3848'], - ], - }, - { - metric: { - __name__: 'test2', - instance: 'localhost:8080', - job: 'otherjob', - }, - values: [ - [1443454529, '3847'], - [1443454531, '3849'], - ], + format: 'time_series', + refId: 'A', }, ], - }, - }; - - it('should return data frame', () => { - const result = transform({ data: response } as any, { - ...options, - target: { - responseListLength: 0, - refId: 'A', - format: 'table', - }, - }); - expect(result[0].fields[0].values.toArray()).toEqual([ - 1443454528000, - 1443454530000, - 1443454529000, - 1443454531000, - ]); - expect(result[0].fields[0].name).toBe('Time'); - expect(result[0].fields[0].type).toBe(FieldType.time); - expect(result[0].fields[1].values.toArray()).toEqual(['test', 'test', 'test2', 'test2']); - expect(result[0].fields[1].name).toBe('__name__'); - expect(result[0].fields[1].config.filterable).toBe(true); - expect(result[0].fields[1].type).toBe(FieldType.string); - expect(result[0].fields[2].values.toArray()).toEqual(['', '', 'localhost:8080', 'localhost:8080']); - expect(result[0].fields[2].name).toBe('instance'); - expect(result[0].fields[2].type).toBe(FieldType.string); - expect(result[0].fields[3].values.toArray()).toEqual(['testjob', 'testjob', 'otherjob', 'otherjob']); - expect(result[0].fields[3].name).toBe('job'); - expect(result[0].fields[3].type).toBe(FieldType.string); - expect(result[0].fields[4].values.toArray()).toEqual([3846, 3848, 3847, 3849]); - expect(result[0].fields[4].name).toEqual('Value'); - expect(result[0].fields[4].type).toBe(FieldType.number); - expect(result[0].refId).toBe('A'); - }); - - it('should include refId if response count is more than 2', () => { - const result = transform({ data: response } as any, { - ...options, - target: { - refId: 'B', - format: 'table', - }, - responseListLength: 2, - }); - - expect(result[0].fields[4].name).toEqual('Value #B'); - }); - }); - - describe('When resultFormat is table and instant = true', () => { - const response = { - status: 'success', - data: { - resultType: 'vector', - result: [ - { - metric: { __name__: 'test', job: 'testjob' }, - value: [1443454528, '3846'], - }, - ], - }, - }; - - it('should return data frame', () => { - const result = transform({ data: response } as any, { ...options, target: { format: 'table' } }); - expect(result[0].fields[0].values.toArray()).toEqual([1443454528000]); - expect(result[0].fields[0].name).toBe('Time'); - expect(result[0].fields[1].values.toArray()).toEqual(['test']); - expect(result[0].fields[1].name).toBe('__name__'); - expect(result[0].fields[2].values.toArray()).toEqual(['testjob']); - expect(result[0].fields[2].name).toBe('job'); - expect(result[0].fields[3].values.toArray()).toEqual([3846]); - expect(result[0].fields[3].name).toEqual('Value'); - }); - - it('should return le label values parsed as numbers', () => { - const response = { - status: 'success', - data: { - resultType: 'vector', - result: [ - { - metric: { le: '102' }, - value: [1594908838, '0'], - }, - ], - }, - }; - const result = transform({ data: response } as any, { ...options, target: { format: 'table' } }); - expect(result[0].fields[1].values.toArray()).toEqual([102]); - expect(result[0].fields[1].type).toEqual(FieldType.number); - }); - }); - - describe('When instant = true', () => { - const response = { - status: 'success', - data: { - resultType: 'vector', - result: [ - { - metric: { __name__: 'test', job: 'testjob' }, - value: [1443454528, '3846'], - }, - ], - }, - }; - - it('should return data frame', () => { - const result: DataFrame[] = transform({ data: response } as any, { ...options, query: { instant: true } }); - expect(result[0].name).toBe('test{job="testjob"}'); - }); - }); - - describe('When resultFormat is heatmap', () => { - const getResponse = (result: any) => ({ - status: 'success', - data: { - resultType: 'matrix', - result, - }, - }); - - const options = { - format: 'heatmap', - start: 1445000010, - end: 1445000030, - legendFormat: '{{le}}', - }; - - it('should convert cumulative histogram to regular', () => { - const response = getResponse([ - { - metric: { __name__: 'test', job: 'testjob', le: '1' }, - values: [ - [1445000010, '10'], - [1445000020, '10'], - [1445000030, '0'], - ], - }, - { - metric: { __name__: 'test', job: 'testjob', le: '2' }, - values: [ - [1445000010, '20'], - [1445000020, '10'], - [1445000030, '30'], - ], - }, - { - metric: { __name__: 'test', job: 'testjob', le: '3' }, - values: [ - [1445000010, '30'], - [1445000020, '10'], - [1445000030, '40'], - ], - }, - ]); - - const result = transform({ data: response } as any, { query: options, target: options } as any); - expect(result[0].fields[0].values.toArray()).toEqual([1445000010000, 1445000020000, 1445000030000]); - expect(result[0].fields[1].values.toArray()).toEqual([10, 10, 0]); - expect(result[1].fields[0].values.toArray()).toEqual([1445000010000, 1445000020000, 1445000030000]); - expect(result[1].fields[1].values.toArray()).toEqual([10, 0, 30]); - expect(result[2].fields[0].values.toArray()).toEqual([1445000010000, 1445000020000, 1445000030000]); - expect(result[2].fields[1].values.toArray()).toEqual([10, 0, 10]); - }); - - it('should handle missing datapoints', () => { - const response = getResponse([ - { - metric: { __name__: 'test', job: 'testjob', le: '1' }, - values: [ - [1445000010, '1'], - [1445000020, '2'], - ], - }, - { - metric: { __name__: 'test', job: 'testjob', le: '2' }, - values: [ - [1445000010, '2'], - [1445000020, '5'], - [1445000030, '1'], - ], - }, - { - metric: { __name__: 'test', job: 'testjob', le: '3' }, - values: [ - [1445000010, '3'], - [1445000020, '7'], - ], - }, - ]); - const result = transform({ data: response } as any, { query: options, target: options } as any); - expect(result[0].fields[1].values.toArray()).toEqual([1, 2]); - expect(result[1].fields[1].values.toArray()).toEqual([1, 3, 1]); - expect(result[2].fields[1].values.toArray()).toEqual([1, 2]); - }); - }); - - describe('When the response is a matrix', () => { - it('should have labels with the value field', () => { - const response = { - status: 'success', - data: { - resultType: 'matrix', - result: [ - { - metric: { __name__: 'test', job: 'testjob', instance: 'testinstance' }, - values: [ - [0, '10'], - [1, '10'], - [2, '0'], - ], - }, - ], - }, - }; - - const result: DataFrame[] = transform({ data: response } as any, { - ...options, - }); - - expect(result[0].fields[1].labels).toBeDefined(); - expect(result[0].fields[1].labels?.instance).toBe('testinstance'); - expect(result[0].fields[1].labels?.job).toBe('testjob'); - }); - - it('should transform into a data frame', () => { - const response = { - status: 'success', - data: { - resultType: 'matrix', - result: [ - { - metric: { __name__: 'test', job: 'testjob' }, - values: [ - [0, '10'], - [1, '10'], - [2, '0'], - ], - }, - ], - }, - }; - - const result: DataFrame[] = transform({ data: response } as any, { - ...options, - query: { - start: 0, - end: 2, - }, - }); - expect(result[0].fields[0].values.toArray()).toEqual([0, 1000, 2000]); - expect(result[0].fields[1].values.toArray()).toEqual([10, 10, 0]); - expect(result[0].name).toBe('test{job="testjob"}'); - }); - - it('should fill null values', () => { - const result = transform({ data: matrixResponse } as any, { ...options, query: { step: 1, start: 0, end: 2 } }); - - expect(result[0].fields[0].values.toArray()).toEqual([0, 1000, 2000]); - expect(result[0].fields[1].values.toArray()).toEqual([null, 10, 0]); - }); - - it('should use __name__ label as series name', () => { - const result = transform({ data: matrixResponse } as any, { - ...options, - query: { - step: 1, - start: 0, - end: 2, - }, - }); - expect(result[0].name).toEqual('test{job="testjob"}'); - }); - - it('should use query as series name when __name__ is not available and metric is empty', () => { - const response = { - status: 'success', - data: { - resultType: 'matrix', - result: [ - { - metric: {}, - values: [[0, '10']], - }, - ], - }, - }; - const expr = 'histogram_quantile(0.95, sum(rate(tns_request_duration_seconds_bucket[5m])) by (le))'; - const result = transform({ data: response } as any, { - ...options, - query: { - step: 1, - start: 0, - end: 2, - expr, - }, - }); - expect(result[0].name).toEqual(expr); - }); - - it('should set frame name to undefined if no __name__ label but there are other labels', () => { - const response = { - status: 'success', - data: { - resultType: 'matrix', - result: [ - { - metric: { job: 'testjob' }, - values: [ - [1, '10'], - [2, '0'], - ], - }, - ], - }, - }; - - const result = transform({ data: response } as any, { - ...options, - query: { - step: 1, - start: 0, - end: 2, - }, - }); - expect(result[0].name).toBe('{job="testjob"}'); - }); - - it('should not set displayName for ValueFields', () => { - const result = transform({ data: matrixResponse } as any, options); - expect(result[0].fields[1].config.displayName).toBeUndefined(); - expect(result[0].fields[1].config.displayNameFromDS).toBe('test{job="testjob"}'); - }); - - it('should align null values with step', () => { - const response = { - status: 'success', - data: { - resultType: 'matrix', - result: [ - { - metric: { __name__: 'test', job: 'testjob' }, - values: [ - [4, '10'], - [8, '10'], - ], - }, - ], - }, - }; - - const result = transform({ data: response } as any, { ...options, query: { step: 2, start: 0, end: 8 } }); - expect(result[0].fields[0].values.toArray()).toEqual([0, 2000, 4000, 6000, 8000]); - expect(result[0].fields[1].values.toArray()).toEqual([null, null, 10, null, 10]); - }); - }); - - describe('When infinity values are returned', () => { - describe('When resultType is scalar', () => { - const response = { - status: 'success', - data: { - resultType: 'scalar', - result: [1443454528, '+Inf'], - }, - }; - - it('should correctly parse values', () => { - const result: DataFrame[] = transform({ data: response } as any, { ...options, target: { format: 'table' } }); - expect(result[0].fields[1].values.toArray()).toEqual([Number.POSITIVE_INFINITY]); - }); - }); - - describe('When resultType is vector', () => { - const response = { - status: 'success', - data: { - resultType: 'vector', - result: [ - { - metric: { __name__: 'test', job: 'testjob' }, - value: [1443454528, '+Inf'], - }, - { - metric: { __name__: 'test', job: 'testjob' }, - value: [1443454528, '-Inf'], - }, - ], - }, - }; - - describe('When format is table', () => { - it('should correctly parse values', () => { - const result: DataFrame[] = transform({ data: response } as any, { ...options, target: { format: 'table' } }); - expect(result[0].fields[3].values.toArray()).toEqual([Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]); - }); - }); - }); - }); - - const exemplarsResponse = { - status: 'success', - data: [ - { - seriesLabels: { __name__: 'test' }, - exemplars: [ - { - timestamp: 1610449069.957, - labels: { traceID: '5020b5bc45117f07' }, - value: 0.002074123, - }, - ], - }, - ], - }; - - describe('When the response is exemplar data', () => { - it('should return as an data frame with a dataTopic annotations', () => { - const result = transform({ data: exemplarsResponse } as any, options); - - expect(result[0].meta?.dataTopic).toBe('annotations'); - expect(result[0].fields.length).toBe(4); // __name__, traceID, Time, Value - expect(result[0].length).toBe(1); - }); - - it('should return with an empty array when data is empty', () => { - const result = transform( - { - data: { - status: 'success', - data: [], - }, - } as any, - options - ); - - expect(result).toHaveLength(0); - }); - - it('should remove exemplars that are too close to each other', () => { - const response = { - status: 'success', + } as unknown) as DataQueryRequest; + const response = ({ + state: 'Done', data: [ { - exemplars: [ + fields: [], + length: 2, + name: 'ALERTS', + refId: 'A', + }, + ], + } as unknown) as DataQueryResponse; + const series = transformV2(response, options); + expect(series).toEqual({ + data: [{ fields: [], length: 2, meta: { preferredVisualisationType: 'graph' }, name: 'ALERTS', refId: 'A' }], + state: 'Done', + }); + }); + + it('results with table format should be transformed to table dataFrames', () => { + const options = ({ + targets: [ + { + format: 'table', + refId: 'A', + }, + ], + } as unknown) as DataQueryRequest; + const response = ({ + state: 'Done', + data: [ + new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [6, 5, 4] }, { - timestamp: 1610449070.0, - value: 5, + name: 'value', + type: FieldType.number, + values: [6, 5, 4], + labels: { label1: 'value1', label2: 'value2' }, }, + ], + }), + ], + } as unknown) as DataQueryResponse; + const series = transformV2(response, options); + // expect(series.data[0]).toBe({}); + expect(series.data[0].fields[0].name).toEqual('time'); + expect(series.data[0].fields[1].name).toEqual('label1'); + expect(series.data[0].fields[2].name).toEqual('label2'); + expect(series.data[0].fields[3].name).toEqual('Value'); + expect(series.data[0].meta?.preferredVisualisationType).toEqual('table'); + }); + + it('results with table and time_series format should be correctly transformed', () => { + const options = ({ + targets: [ + { + format: 'table', + refId: 'A', + }, + { + format: 'time_series', + refId: 'B', + }, + ], + } as unknown) as DataQueryRequest; + const response = ({ + state: 'Done', + data: [ + new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [6, 5, 4] }, { - timestamp: 1610449070.0, - value: 1, + name: 'value', + type: FieldType.number, + values: [6, 5, 4], + labels: { label1: 'value1', label2: 'value2' }, }, + ], + }), + new MutableDataFrame({ + refId: 'B', + fields: [ + { name: 'time', type: FieldType.time, values: [6, 5, 4] }, { - timestamp: 1610449070.5, - value: 13, + name: 'value', + type: FieldType.number, + values: [6, 5, 4], + labels: { label1: 'value1', label2: 'value2' }, }, + ], + }), + ], + } as unknown) as DataQueryResponse; + const series = transformV2(response, options); + expect(series.data[0].fields.length).toEqual(2); + expect(series.data[0].meta?.preferredVisualisationType).toEqual('graph'); + expect(series.data[1].fields.length).toEqual(4); + expect(series.data[1].meta?.preferredVisualisationType).toEqual('table'); + }); + }); + describe('transformDFoTable', () => { + it('transforms dataFrame with response length 1 to table dataFrame', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [6, 5, 4] }, + { + name: 'value', + type: FieldType.number, + values: [6, 5, 4], + labels: { label1: 'value1', label2: 'value2' }, + }, + ], + }); + + const tableDf = transformDFoTable(df, 1); + expect(tableDf.fields.length).toBe(4); + expect(tableDf.fields[0].name).toBe('time'); + expect(tableDf.fields[1].name).toBe('label1'); + expect(tableDf.fields[1].values.get(0)).toBe('value1'); + expect(tableDf.fields[2].name).toBe('label2'); + expect(tableDf.fields[2].values.get(0)).toBe('value2'); + expect(tableDf.fields[3].name).toBe('Value'); + }); + + it('transforms dataFrame with response length 2 to table dataFrame', () => { + const df = new MutableDataFrame({ + refId: 'A', + fields: [ + { name: 'time', type: FieldType.time, values: [6, 5, 4] }, + { + name: 'value', + type: FieldType.number, + values: [6, 5, 4], + labels: { label1: 'value1', label2: 'value2' }, + }, + ], + }); + + const tableDf = transformDFoTable(df, 3); + expect(tableDf.fields.length).toBe(4); + expect(tableDf.fields[0].name).toBe('time'); + expect(tableDf.fields[1].name).toBe('label1'); + expect(tableDf.fields[1].values.get(0)).toBe('value1'); + expect(tableDf.fields[2].name).toBe('label2'); + expect(tableDf.fields[2].values.get(0)).toBe('value2'); + expect(tableDf.fields[3].name).toBe('Value #A'); + }); + }); + + describe('transform', () => { + const options: any = { target: {}, query: {} }; + describe('When nothing is returned', () => { + it('should return empty array', () => { + const response = { + status: 'success', + data: { + resultType: '', + result: null, + }, + }; + const series = transform({ data: response } as any, options); + expect(series).toEqual([]); + }); + it('should return empty array', () => { + const response = { + status: 'success', + data: { + resultType: '', + result: null, + }, + }; + const result = transform({ data: response } as any, { ...options, target: { format: 'table' } }); + expect(result).toHaveLength(0); + }); + }); + + describe('When resultFormat is table', () => { + const response = { + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: { __name__: 'test', job: 'testjob' }, + values: [ + [1443454528, '3846'], + [1443454530, '3848'], + ], + }, + { + metric: { + __name__: 'test2', + instance: 'localhost:8080', + job: 'otherjob', + }, + values: [ + [1443454529, '3847'], + [1443454531, '3849'], + ], + }, + ], + }, + }; + + it('should return data frame', () => { + const result = transform({ data: response } as any, { + ...options, + target: { + responseListLength: 0, + refId: 'A', + format: 'table', + }, + }); + expect(result[0].fields[0].values.toArray()).toEqual([ + 1443454528000, + 1443454530000, + 1443454529000, + 1443454531000, + ]); + expect(result[0].fields[0].name).toBe('Time'); + expect(result[0].fields[0].type).toBe(FieldType.time); + expect(result[0].fields[1].values.toArray()).toEqual(['test', 'test', 'test2', 'test2']); + expect(result[0].fields[1].name).toBe('__name__'); + expect(result[0].fields[1].config.filterable).toBe(true); + expect(result[0].fields[1].type).toBe(FieldType.string); + expect(result[0].fields[2].values.toArray()).toEqual(['', '', 'localhost:8080', 'localhost:8080']); + expect(result[0].fields[2].name).toBe('instance'); + expect(result[0].fields[2].type).toBe(FieldType.string); + expect(result[0].fields[3].values.toArray()).toEqual(['testjob', 'testjob', 'otherjob', 'otherjob']); + expect(result[0].fields[3].name).toBe('job'); + expect(result[0].fields[3].type).toBe(FieldType.string); + expect(result[0].fields[4].values.toArray()).toEqual([3846, 3848, 3847, 3849]); + expect(result[0].fields[4].name).toEqual('Value'); + expect(result[0].fields[4].type).toBe(FieldType.number); + expect(result[0].refId).toBe('A'); + }); + + it('should include refId if response count is more than 2', () => { + const result = transform({ data: response } as any, { + ...options, + target: { + refId: 'B', + format: 'table', + }, + responseListLength: 2, + }); + + expect(result[0].fields[4].name).toEqual('Value #B'); + }); + }); + + describe('When resultFormat is table and instant = true', () => { + const response = { + status: 'success', + data: { + resultType: 'vector', + result: [ + { + metric: { __name__: 'test', job: 'testjob' }, + value: [1443454528, '3846'], + }, + ], + }, + }; + + it('should return data frame', () => { + const result = transform({ data: response } as any, { ...options, target: { format: 'table' } }); + expect(result[0].fields[0].values.toArray()).toEqual([1443454528000]); + expect(result[0].fields[0].name).toBe('Time'); + expect(result[0].fields[1].values.toArray()).toEqual(['test']); + expect(result[0].fields[1].name).toBe('__name__'); + expect(result[0].fields[2].values.toArray()).toEqual(['testjob']); + expect(result[0].fields[2].name).toBe('job'); + expect(result[0].fields[3].values.toArray()).toEqual([3846]); + expect(result[0].fields[3].name).toEqual('Value'); + }); + + it('should return le label values parsed as numbers', () => { + const response = { + status: 'success', + data: { + resultType: 'vector', + result: [ { - timestamp: 1610449070.3, - value: 20, + metric: { le: '102' }, + value: [1594908838, '0'], }, ], }, - ], - }; - /** - * the standard deviation for the above values is 8.4 this means that we show the highest - * value (20) and then the next value should be 2 times the standard deviation which is 1 - **/ - const result = transform({ data: response } as any, options); - expect(result[0].length).toBe(2); + }; + const result = transform({ data: response } as any, { ...options, target: { format: 'table' } }); + expect(result[0].fields[1].values.toArray()).toEqual([102]); + expect(result[0].fields[1].type).toEqual(FieldType.number); + }); }); - describe('data link', () => { - it('should be added to the field if found with url', () => { - const result = transform({ data: exemplarsResponse } as any, { - ...options, - exemplarTraceIdDestinations: [{ name: 'traceID', url: 'http://localhost' }], - }); + describe('When instant = true', () => { + const response = { + status: 'success', + data: { + resultType: 'vector', + result: [ + { + metric: { __name__: 'test', job: 'testjob' }, + value: [1443454528, '3846'], + }, + ], + }, + }; - expect(result[0].fields.some((f) => f.config.links?.length)).toBe(true); + it('should return data frame', () => { + const result: DataFrame[] = transform({ data: response } as any, { ...options, query: { instant: true } }); + expect(result[0].name).toBe('test{job="testjob"}'); + }); + }); + + describe('When resultFormat is heatmap', () => { + const getResponse = (result: any) => ({ + status: 'success', + data: { + resultType: 'matrix', + result, + }, }); - it('should be added to the field if found with internal link', () => { - const result = transform({ data: exemplarsResponse } as any, { - ...options, - exemplarTraceIdDestinations: [{ name: 'traceID', datasourceUid: 'jaeger' }], - }); + const options = { + format: 'heatmap', + start: 1445000010, + end: 1445000030, + legendFormat: '{{le}}', + }; - expect(result[0].fields.some((f) => f.config.links?.length)).toBe(true); + it('should convert cumulative histogram to regular', () => { + const response = getResponse([ + { + metric: { __name__: 'test', job: 'testjob', le: '1' }, + values: [ + [1445000010, '10'], + [1445000020, '10'], + [1445000030, '0'], + ], + }, + { + metric: { __name__: 'test', job: 'testjob', le: '2' }, + values: [ + [1445000010, '20'], + [1445000020, '10'], + [1445000030, '30'], + ], + }, + { + metric: { __name__: 'test', job: 'testjob', le: '3' }, + values: [ + [1445000010, '30'], + [1445000020, '10'], + [1445000030, '40'], + ], + }, + ]); + + const result = transform({ data: response } as any, { query: options, target: options } as any); + expect(result[0].fields[0].values.toArray()).toEqual([1445000010000, 1445000020000, 1445000030000]); + expect(result[0].fields[1].values.toArray()).toEqual([10, 10, 0]); + expect(result[1].fields[0].values.toArray()).toEqual([1445000010000, 1445000020000, 1445000030000]); + expect(result[1].fields[1].values.toArray()).toEqual([10, 0, 30]); + expect(result[2].fields[0].values.toArray()).toEqual([1445000010000, 1445000020000, 1445000030000]); + expect(result[2].fields[1].values.toArray()).toEqual([10, 0, 10]); }); - it('should not add link if exemplarTraceIdDestinations is not configured', () => { + it('should handle missing datapoints', () => { + const response = getResponse([ + { + metric: { __name__: 'test', job: 'testjob', le: '1' }, + values: [ + [1445000010, '1'], + [1445000020, '2'], + ], + }, + { + metric: { __name__: 'test', job: 'testjob', le: '2' }, + values: [ + [1445000010, '2'], + [1445000020, '5'], + [1445000030, '1'], + ], + }, + { + metric: { __name__: 'test', job: 'testjob', le: '3' }, + values: [ + [1445000010, '3'], + [1445000020, '7'], + ], + }, + ]); + const result = transform({ data: response } as any, { query: options, target: options } as any); + expect(result[0].fields[1].values.toArray()).toEqual([1, 2]); + expect(result[1].fields[1].values.toArray()).toEqual([1, 3, 1]); + expect(result[2].fields[1].values.toArray()).toEqual([1, 2]); + }); + }); + + describe('When the response is a matrix', () => { + it('should have labels with the value field', () => { + const response = { + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: { __name__: 'test', job: 'testjob', instance: 'testinstance' }, + values: [ + [0, '10'], + [1, '10'], + [2, '0'], + ], + }, + ], + }, + }; + + const result: DataFrame[] = transform({ data: response } as any, { + ...options, + }); + + expect(result[0].fields[1].labels).toBeDefined(); + expect(result[0].fields[1].labels?.instance).toBe('testinstance'); + expect(result[0].fields[1].labels?.job).toBe('testjob'); + }); + + it('should transform into a data frame', () => { + const response = { + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: { __name__: 'test', job: 'testjob' }, + values: [ + [0, '10'], + [1, '10'], + [2, '0'], + ], + }, + ], + }, + }; + + const result: DataFrame[] = transform({ data: response } as any, { + ...options, + query: { + start: 0, + end: 2, + }, + }); + expect(result[0].fields[0].values.toArray()).toEqual([0, 1000, 2000]); + expect(result[0].fields[1].values.toArray()).toEqual([10, 10, 0]); + expect(result[0].name).toBe('test{job="testjob"}'); + }); + + it('should fill null values', () => { + const result = transform({ data: matrixResponse } as any, { + ...options, + query: { step: 1, start: 0, end: 2 }, + }); + + expect(result[0].fields[0].values.toArray()).toEqual([0, 1000, 2000]); + expect(result[0].fields[1].values.toArray()).toEqual([null, 10, 0]); + }); + + it('should use __name__ label as series name', () => { + const result = transform({ data: matrixResponse } as any, { + ...options, + query: { + step: 1, + start: 0, + end: 2, + }, + }); + expect(result[0].name).toEqual('test{job="testjob"}'); + }); + + it('should use query as series name when __name__ is not available and metric is empty', () => { + const response = { + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: {}, + values: [[0, '10']], + }, + ], + }, + }; + const expr = 'histogram_quantile(0.95, sum(rate(tns_request_duration_seconds_bucket[5m])) by (le))'; + const result = transform({ data: response } as any, { + ...options, + query: { + step: 1, + start: 0, + end: 2, + expr, + }, + }); + expect(result[0].name).toEqual(expr); + }); + + it('should set frame name to undefined if no __name__ label but there are other labels', () => { + const response = { + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: { job: 'testjob' }, + values: [ + [1, '10'], + [2, '0'], + ], + }, + ], + }, + }; + + const result = transform({ data: response } as any, { + ...options, + query: { + step: 1, + start: 0, + end: 2, + }, + }); + expect(result[0].name).toBe('{job="testjob"}'); + }); + + it('should not set displayName for ValueFields', () => { + const result = transform({ data: matrixResponse } as any, options); + expect(result[0].fields[1].config.displayName).toBeUndefined(); + expect(result[0].fields[1].config.displayNameFromDS).toBe('test{job="testjob"}'); + }); + + it('should align null values with step', () => { + const response = { + status: 'success', + data: { + resultType: 'matrix', + result: [ + { + metric: { __name__: 'test', job: 'testjob' }, + values: [ + [4, '10'], + [8, '10'], + ], + }, + ], + }, + }; + + const result = transform({ data: response } as any, { ...options, query: { step: 2, start: 0, end: 8 } }); + expect(result[0].fields[0].values.toArray()).toEqual([0, 2000, 4000, 6000, 8000]); + expect(result[0].fields[1].values.toArray()).toEqual([null, null, 10, null, 10]); + }); + }); + + describe('When infinity values are returned', () => { + describe('When resultType is scalar', () => { + const response = { + status: 'success', + data: { + resultType: 'scalar', + result: [1443454528, '+Inf'], + }, + }; + + it('should correctly parse values', () => { + const result: DataFrame[] = transform({ data: response } as any, { + ...options, + target: { format: 'table' }, + }); + expect(result[0].fields[1].values.toArray()).toEqual([Number.POSITIVE_INFINITY]); + }); + }); + + describe('When resultType is vector', () => { + const response = { + status: 'success', + data: { + resultType: 'vector', + result: [ + { + metric: { __name__: 'test', job: 'testjob' }, + value: [1443454528, '+Inf'], + }, + { + metric: { __name__: 'test', job: 'testjob' }, + value: [1443454528, '-Inf'], + }, + ], + }, + }; + + describe('When format is table', () => { + it('should correctly parse values', () => { + const result: DataFrame[] = transform({ data: response } as any, { + ...options, + target: { format: 'table' }, + }); + expect(result[0].fields[3].values.toArray()).toEqual([Number.POSITIVE_INFINITY, Number.NEGATIVE_INFINITY]); + }); + }); + }); + }); + + const exemplarsResponse = { + status: 'success', + data: [ + { + seriesLabels: { __name__: 'test' }, + exemplars: [ + { + timestamp: 1610449069.957, + labels: { traceID: '5020b5bc45117f07' }, + value: 0.002074123, + }, + ], + }, + ], + }; + + describe('When the response is exemplar data', () => { + it('should return as an data frame with a dataTopic annotations', () => { const result = transform({ data: exemplarsResponse } as any, options); - expect(result[0].fields.some((f) => f.config.links?.length)).toBe(false); + expect(result[0].meta?.dataTopic).toBe('annotations'); + expect(result[0].fields.length).toBe(4); // __name__, traceID, Time, Value + expect(result[0].length).toBe(1); + }); + + it('should return with an empty array when data is empty', () => { + const result = transform( + { + data: { + status: 'success', + data: [], + }, + } as any, + options + ); + + expect(result).toHaveLength(0); + }); + + it('should remove exemplars that are too close to each other', () => { + const response = { + status: 'success', + data: [ + { + exemplars: [ + { + timestamp: 1610449070.0, + value: 5, + }, + { + timestamp: 1610449070.0, + value: 1, + }, + { + timestamp: 1610449070.5, + value: 13, + }, + { + timestamp: 1610449070.3, + value: 20, + }, + ], + }, + ], + }; + /** + * the standard deviation for the above values is 8.4 this means that we show the highest + * value (20) and then the next value should be 2 times the standard deviation which is 1 + **/ + const result = transform({ data: response } as any, options); + expect(result[0].length).toBe(2); + }); + + describe('data link', () => { + it('should be added to the field if found with url', () => { + const result = transform({ data: exemplarsResponse } as any, { + ...options, + exemplarTraceIdDestinations: [{ name: 'traceID', url: 'http://localhost' }], + }); + + expect(result[0].fields.some((f) => f.config.links?.length)).toBe(true); + }); + + it('should be added to the field if found with internal link', () => { + const result = transform({ data: exemplarsResponse } as any, { + ...options, + exemplarTraceIdDestinations: [{ name: 'traceID', datasourceUid: 'jaeger' }], + }); + + expect(result[0].fields.some((f) => f.config.links?.length)).toBe(true); + }); + + it('should not add link if exemplarTraceIdDestinations is not configured', () => { + const result = transform({ data: exemplarsResponse } as any, options); + + expect(result[0].fields.some((f) => f.config.links?.length)).toBe(false); + }); }); }); }); diff --git a/public/app/plugins/datasource/prometheus/result_transformer.ts b/public/app/plugins/datasource/prometheus/result_transformer.ts index ae55bcff9ac..931c80dc782 100644 --- a/public/app/plugins/datasource/prometheus/result_transformer.ts +++ b/public/app/plugins/datasource/prometheus/result_transformer.ts @@ -13,8 +13,12 @@ import { ScopedVars, TIME_SERIES_TIME_FIELD_NAME, TIME_SERIES_VALUE_FIELD_NAME, + DataQueryResponse, + DataQueryRequest, + PreferredVisualisationType, } from '@grafana/data'; import { FetchResponse, getDataSourceSrv, getTemplateSrv } from '@grafana/runtime'; +import { partition } from 'lodash'; import { descending, deviation } from 'd3'; import { ExemplarTraceIdDestination, @@ -37,6 +41,84 @@ interface TimeAndValue { [TIME_SERIES_VALUE_FIELD_NAME]: number; } +// V2 result trasnformer used to transform query results from queries that were run trough prometheus backend +export function transformV2(response: DataQueryResponse, options: DataQueryRequest) { + // Get refIds that have table format as we need to process those to table reuslts + const tableRefIds = options.targets.filter((target) => target.format === 'table').map((target) => target.refId); + const [tableResults, otherResults]: [DataFrame[], DataFrame[]] = partition(response.data, (dataFrame) => + dataFrame.refId ? tableRefIds.includes(dataFrame.refId) : false + ); + + // For table results, we need to transform data frames to table data frames + const tableFrames = tableResults.map((dataFrame) => { + const df = transformDFoTable(dataFrame, options.targets.length); + return df; + }); + + // Everything else is processed as time_series result and graph preferredVisualisationType + const otherFrames = otherResults.map((dataFrame) => { + const df = { + ...dataFrame, + meta: { + ...dataFrame.meta, + preferredVisualisationType: 'graph', + }, + } as DataFrame; + return df; + }); + + return { ...response, data: [...otherFrames, ...tableFrames] }; +} + +export function transformDFoTable(df: DataFrame, responseLength: number): DataFrame { + if (df.length === 0) { + return df; + } + + const timeField = df.fields[0]; + const valueField = df.fields[1]; + + // Create label fields + const promLabels: PromMetric = valueField.labels ?? {}; + const labelFields = Object.keys(promLabels) + .sort() + .map((label) => { + const numberField = label === 'le'; + return { + name: label, + config: { filterable: true }, + type: numberField ? FieldType.number : FieldType.string, + values: new ArrayVector(), + }; + }); + + // Fill labelFields with label values + labelFields.forEach((field) => field.values.add(getLabelValue(promLabels, field.name))); + + const tableDataFrame = { + ...df, + name: undefined, + meta: { ...df.meta, preferredVisualisationType: 'table' as PreferredVisualisationType }, + fields: [ + timeField, + ...labelFields, + { + ...valueField, + name: getValueText(responseLength, df.refId), + labels: undefined, + config: { ...valueField.config, displayNameFromDS: undefined }, + state: { ...valueField.state, displayName: undefined }, + }, + ], + }; + + return tableDataFrame; +} + +function getValueText(responseLength: number, refId = '') { + return responseLength > 1 ? `Value #${refId}` : 'Value'; +} + export function transform( response: FetchResponse, transformOptions: { diff --git a/public/app/plugins/datasource/prometheus/types.ts b/public/app/plugins/datasource/prometheus/types.ts index cdf857d0b8d..9c391287d8d 100644 --- a/public/app/plugins/datasource/prometheus/types.ts +++ b/public/app/plugins/datasource/prometheus/types.ts @@ -10,6 +10,8 @@ export interface PromQuery extends DataQuery { hinting?: boolean; interval?: string; intervalFactor?: number; + // Timezone offset to align start & end time on backend + offsetSec?: number; legendFormat?: string; valueWithRefId?: boolean; requestId?: string;