From 861eb72113b03ffa3e8f8c0ddafe7c2355ab2715 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Wed, 30 Oct 2019 11:38:28 -0700 Subject: [PATCH] transform: add expressions to query editor (w/ feature flag) (#20072) for use with gel which is not released yet. --- package.json | 1 + packages/grafana-runtime/src/config.ts | 2 + pkg/api/api.go | 1 - pkg/api/metrics.go | 62 ++++--- pkg/api/transform.go | 79 --------- pkg/plugins/transform_plugin.go | 20 +-- .../dashboard/panel_editor/QueriesTab.tsx | 23 ++- .../features/dashboard/state/runRequest.ts | 11 ++ .../expressions/ExpressionDatasource.ts | 84 +++++++++ .../expressions/ExpressionQueryEditor.tsx | 159 ++++++++++++++++++ .../__snapshots__/util.test.ts.snap | 128 ++++++++++++++ public/app/features/expressions/types.ts | 20 +++ public/app/features/expressions/util.test.ts | 48 ++++++ public/app/features/expressions/util.ts | 62 +++++++ public/app/features/plugins/datasource_srv.ts | 9 + yarn.lock | 143 +++++++++++++++- 16 files changed, 726 insertions(+), 126 deletions(-) delete mode 100644 pkg/api/transform.go create mode 100644 public/app/features/expressions/ExpressionDatasource.ts create mode 100644 public/app/features/expressions/ExpressionQueryEditor.tsx create mode 100644 public/app/features/expressions/__snapshots__/util.test.ts.snap create mode 100644 public/app/features/expressions/types.ts create mode 100644 public/app/features/expressions/util.test.ts create mode 100644 public/app/features/expressions/util.ts diff --git a/package.json b/package.json index 61fb9e1c0e6..e66b8d23233 100644 --- a/package.json +++ b/package.json @@ -203,6 +203,7 @@ "angular-native-dragdrop": "1.2.2", "angular-route": "1.6.6", "angular-sanitize": "1.6.6", + "apache-arrow": "0.15.0", "baron": "3.0.3", "brace": "0.10.0", "calculate-size": "1.1.1", diff --git a/packages/grafana-runtime/src/config.ts b/packages/grafana-runtime/src/config.ts index f329549ff6a..19d90e5a379 100644 --- a/packages/grafana-runtime/src/config.ts +++ b/packages/grafana-runtime/src/config.ts @@ -12,6 +12,7 @@ export interface BuildInfo { interface FeatureToggles { transformations: boolean; + expressions: boolean; } export class GrafanaBootConfig { datasources: { [str: string]: DataSourceInstanceSettings } = {}; @@ -46,6 +47,7 @@ export class GrafanaBootConfig { pluginsToPreload: string[] = []; featureToggles: FeatureToggles = { transformations: false, + expressions: false, }; constructor(options: GrafanaBootConfig) { diff --git a/pkg/api/api.go b/pkg/api/api.go index d5b6b47dce0..9308403b276 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -328,7 +328,6 @@ func (hs *HTTPServer) registerRoutes() { // metrics apiRoute.Post("/tsdb/query", bind(dtos.MetricRequest{}), Wrap(hs.QueryMetrics)) apiRoute.Post("/tsdb/query/v2", bind(dtos.MetricRequest{}), Wrap(hs.QueryMetricsV2)) - apiRoute.Post("/tsdb/transform", bind(dtos.MetricRequest{}), Wrap(hs.Transform)) apiRoute.Get("/tsdb/testdata/scenarios", Wrap(GetTestDataScenarios)) apiRoute.Get("/tsdb/testdata/gensql", reqGrafanaAdmin, Wrap(GenerateSQLTestData)) apiRoute.Get("/tsdb/testdata/random-walk", Wrap(GetTestDataRandomWalk)) diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go index 4a0a90c1030..7889fb9ec94 100644 --- a/pkg/api/metrics.go +++ b/pkg/api/metrics.go @@ -4,6 +4,7 @@ import ( "context" "sort" + "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/api/dtos" @@ -21,37 +22,40 @@ func (hs *HTTPServer) QueryMetricsV2(c *m.ReqContext, reqDto dtos.MetricRequest) return Error(404, "Expressions feature toggle is not enabled", nil) } - timeRange := tsdb.NewTimeRange(reqDto.From, reqDto.To) - if len(reqDto.Queries) == 0 { - return Error(400, "No queries found in query", nil) + return Error(500, "No queries found in query", nil) } - var datasourceID int64 - for _, query := range reqDto.Queries { + request := &tsdb.TsdbQuery{ + TimeRange: tsdb.NewTimeRange(reqDto.From, reqDto.To), + Debug: reqDto.Debug, + } + + expr := false + var ds *m.DataSource + for i, query := range reqDto.Queries { name, err := query.Get("datasource").String() if err != nil { return Error(500, "datasource missing name", err) } - datasourceID, err = query.Get("datasourceId").Int64() + if name == "__expr__" { + expr = true + } + + datasourceID, err := query.Get("datasourceId").Int64() if err != nil { - return Error(400, "GEL datasource missing ID", nil) + return Error(500, "datasource missing ID", nil) } - if name == "-- GEL --" { - break - } - } - ds, err := hs.DatasourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache) - if err != nil { - if err == m.ErrDataSourceAccessDenied { - return Error(403, "Access denied to datasource", err) - } - return Error(500, "Unable to load datasource meta data", err) - } - request := &tsdb.TsdbQuery{TimeRange: timeRange, Debug: reqDto.Debug} - - for _, query := range reqDto.Queries { + if i == 0 && !expr { + ds, err = hs.DatasourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache) + if err != nil { + if err == m.ErrDataSourceAccessDenied { + return Error(403, "Access denied to datasource", err) + } + return Error(500, "Unable to load datasource meta data", err) + } + } request.Queries = append(request.Queries, &tsdb.Query{ RefId: query.Get("refId").MustString("A"), MaxDataPoints: query.Get("maxDataPoints").MustInt64(100), @@ -59,11 +63,21 @@ func (hs *HTTPServer) QueryMetricsV2(c *m.ReqContext, reqDto dtos.MetricRequest) Model: query, DataSource: ds, }) + } - resp, err := tsdb.HandleRequest(c.Req.Context(), ds, request) - if err != nil { - return Error(500, "Metric request error", err) + var resp *tsdb.Response + var err error + if !expr { + resp, err = tsdb.HandleRequest(c.Req.Context(), ds, request) + if err != nil { + return Error(500, "Metric request error", err) + } + } else { + resp, err = plugins.Transform.Transform(c.Req.Context(), request) + if err != nil { + return Error(500, "Transform request error", err) + } } statusCode := 200 diff --git a/pkg/api/transform.go b/pkg/api/transform.go deleted file mode 100644 index 789c6fcbf03..00000000000 --- a/pkg/api/transform.go +++ /dev/null @@ -1,79 +0,0 @@ -package api - -import ( - "net/http" - - "github.com/grafana/grafana/pkg/api/dtos" - m "github.com/grafana/grafana/pkg/models" - "github.com/grafana/grafana/pkg/plugins" - "github.com/grafana/grafana/pkg/setting" - "github.com/grafana/grafana/pkg/tsdb" -) - -// POST /api/tsdb/transform -// This enpoint is tempory, will be part of v2 query endpoint. -func (hs *HTTPServer) Transform(c *m.ReqContext, reqDto dtos.MetricRequest) Response { - if !setting.IsExpressionsEnabled() { - return Error(404, "Expressions feature toggle is not enabled", nil) - } - if plugins.Transform == nil { - return Error(http.StatusServiceUnavailable, "transform plugin is not loaded", nil) - } - - timeRange := tsdb.NewTimeRange(reqDto.From, reqDto.To) - - if len(reqDto.Queries) == 0 { - return Error(400, "No queries found in query", nil) - } - - var datasourceID int64 - for _, query := range reqDto.Queries { - name, err := query.Get("datasource").String() - if err != nil { - return Error(500, "datasource missing name", err) - } - datasourceID, err = query.Get("datasourceId").Int64() - if err != nil { - return Error(400, "GEL datasource missing ID", nil) - } - if name == "-- GEL --" { - break - } - } - - ds, err := hs.DatasourceCache.GetDatasource(datasourceID, c.SignedInUser, c.SkipCache) - if err != nil { - if err == m.ErrDataSourceAccessDenied { - return Error(403, "Access denied to datasource", err) - } - return Error(500, "Unable to load datasource meta data", err) - } - - request := &tsdb.TsdbQuery{TimeRange: timeRange, Debug: reqDto.Debug} - - for _, query := range reqDto.Queries { - request.Queries = append(request.Queries, &tsdb.Query{ - RefId: query.Get("refId").MustString("A"), - MaxDataPoints: query.Get("maxDataPoints").MustInt64(100), - IntervalMs: query.Get("intervalMs").MustInt64(1000), - Model: query, - DataSource: ds, - }) - } - - resp, err := plugins.Transform.Transform(c.Req.Context(), ds, request) - if err != nil { - return Error(500, "Transform request error", err) - } - - statusCode := 200 - for _, res := range resp.Results { - if res.Error != nil { - res.ErrorString = res.Error.Error() - resp.Message = res.ErrorString - statusCode = 400 - } - } - - return JSON(statusCode, &resp) -} diff --git a/pkg/plugins/transform_plugin.go b/pkg/plugins/transform_plugin.go index d96037a62c0..1ef456e666a 100644 --- a/pkg/plugins/transform_plugin.go +++ b/pkg/plugins/transform_plugin.go @@ -27,7 +27,6 @@ type TransformPlugin struct { Executable string `json:"executable,omitempty"` - //transform.TransformPlugin *TransformWrapper client *plugin.Client @@ -136,22 +135,10 @@ type TransformWrapper struct { api *grafanaAPI } -func (tw *TransformWrapper) Transform(ctx context.Context, ds *models.DataSource, query *tsdb.TsdbQuery) (*tsdb.Response, error) { - jsonData, err := ds.JsonData.MarshalJSON() - if err != nil { - return nil, err - } - +func (tw *TransformWrapper) Transform(ctx context.Context, query *tsdb.TsdbQuery) (*tsdb.Response, error) { pbQuery := &pluginv2.TransformRequest{ - Datasource: &pluginv2.DatasourceInfo{ - Name: ds.Name, - Type: ds.Type, - Url: ds.Url, - Id: ds.Id, - OrgId: ds.OrgId, - JsonData: string(jsonData), - DecryptedSecureJsonData: ds.SecureJsonData.Decrypt(), - }, + // TODO Not sure Datasource property needs be on this? + Datasource: &pluginv2.DatasourceInfo{}, TimeRange: &pluginv2.TimeRange{ FromRaw: query.TimeRange.From, ToRaw: query.TimeRange.To, @@ -217,6 +204,7 @@ func (s *grafanaAPI) QueryDatasource(ctx context.Context, req *pluginv2.QueryDat if len(req.Queries) == 0 { return nil, fmt.Errorf("zero queries found in datasource request") } + getDsInfo := &models.GetDataSourceByIdQuery{ Id: req.DatasourceId, OrgId: req.OrgId, diff --git a/public/app/features/dashboard/panel_editor/QueriesTab.tsx b/public/app/features/dashboard/panel_editor/QueriesTab.tsx index b4359ed65ea..5355f181c30 100644 --- a/public/app/features/dashboard/panel_editor/QueriesTab.tsx +++ b/public/app/features/dashboard/panel_editor/QueriesTab.tsx @@ -29,6 +29,7 @@ import { PluginHelp } from 'app/core/components/PluginHelp/PluginHelp'; import { addQuery } from 'app/core/utils/query'; import { Unsubscribable } from 'rxjs'; import { isSharedDashboardQuery, DashboardQueryEditor } from 'app/plugins/datasource/dashboard'; +import { expressionDatasource, ExpressionDatasourceID } from 'app/features/expressions/ExpressionDatasource'; interface Props { panel: PanelModel; @@ -97,9 +98,11 @@ export class QueriesTab extends PureComponent { if (datasource.meta.mixed) { // Set the datasource on all targets panel.targets.forEach(target => { - target.datasource = panel.datasource; - if (!target.datasource) { - target.datasource = config.defaultDatasource; + if (target.datasource !== ExpressionDatasourceID) { + target.datasource = panel.datasource; + if (!target.datasource) { + target.datasource = config.defaultDatasource; + } } }); } else if (currentDS) { @@ -107,7 +110,9 @@ export class QueriesTab extends PureComponent { if (currentDS.meta.mixed) { // Remove the explicit datasource for (const target of panel.targets) { - delete target.datasource; + if (target.datasource !== ExpressionDatasourceID) { + delete target.datasource; + } } } else if (currentDS.meta.id !== datasource.meta.id) { // we are changing data source type, clear queries @@ -150,6 +155,11 @@ export class QueriesTab extends PureComponent { this.onScrollBottom(); }; + onAddExpressionClick = () => { + this.onUpdateQueries(addQuery(this.props.panel.targets, expressionDatasource.newQuery())); + this.onScrollBottom(); + }; + onScrollBottom = () => { this.setState({ scrollTop: this.state.scrollTop + 10000 }); }; @@ -168,6 +178,11 @@ export class QueriesTab extends PureComponent { )} {isAddingMixed && this.renderMixedPicker()} + {config.featureToggles.expressions && ( + + )} ); }; diff --git a/public/app/features/dashboard/state/runRequest.ts b/public/app/features/dashboard/state/runRequest.ts index 7dce1c59eea..3ac44d80a17 100644 --- a/public/app/features/dashboard/state/runRequest.ts +++ b/public/app/features/dashboard/state/runRequest.ts @@ -14,6 +14,7 @@ import { DataQueryError, } from '@grafana/ui'; import { LoadingState, dateMath, toDataFrame, DataFrame, guessFieldTypes } from '@grafana/data'; +import { ExpressionDatasourceID, expressionDatasource } from 'app/features/expressions/ExpressionDatasource'; type MapOfResponsePackets = { [str: string]: DataQueryResponse }; @@ -132,6 +133,16 @@ function cancelNetworkRequestsOnUnsubscribe(req: DataQueryRequest) { } export function callQueryMethod(datasource: DataSourceApi, request: DataQueryRequest) { + console.log('CALL', request.targets); + + // If any query has an expression, use the expression endpoint + for (const target of request.targets) { + if (target.datasource === ExpressionDatasourceID) { + return expressionDatasource.query(request); + } + } + + // Otherwise it is a standard datasource request const returnVal = datasource.query(request); return from(returnVal); } diff --git a/public/app/features/expressions/ExpressionDatasource.ts b/public/app/features/expressions/ExpressionDatasource.ts new file mode 100644 index 00000000000..86870e7b17b --- /dev/null +++ b/public/app/features/expressions/ExpressionDatasource.ts @@ -0,0 +1,84 @@ +import { + DataSourceApi, + DataQueryRequest, + DataQueryResponse, + DataSourceInstanceSettings, + DataSourcePluginMeta, +} from '@grafana/ui'; +import { ExpressionQuery, GELQueryType } from './types'; +import { ExpressionQueryEditor } from './ExpressionQueryEditor'; +import { Observable, from } from 'rxjs'; +import { config } from '@grafana/runtime'; +import { getBackendSrv } from 'app/core/services/backend_srv'; +import { gelResponseToDataFrames } from './util'; + +/** + * This is a singleton that is not actually instanciated + */ +export class ExpressionDatasourceApi extends DataSourceApi { + constructor(instanceSettings: DataSourceInstanceSettings) { + super(instanceSettings); + } + + getCollapsedText(query: ExpressionQuery) { + return `Expression: ${query.type}`; + } + + query(request: DataQueryRequest): Observable { + const { targets, intervalMs, maxDataPoints, range } = request; + + const orgId = (window as any).grafanaBootData.user.orgId; + const queries = targets.map(q => { + if (q.datasource === ExpressionDatasourceID) { + return { + ...q, + datasourceId: this.id, + orgId, + }; + } + const ds = config.datasources[q.datasource || config.defaultDatasource]; + return { + ...q, + datasourceId: ds.id, + intervalMs, + maxDataPoints, + orgId, + // ?? alias: templateSrv.replace(q.alias || ''), + }; + }); + const req: Promise = getBackendSrv() + .post('/api/tsdb/query/v2', { + from: range.from.valueOf().toString(), + to: range.to.valueOf().toString(), + queries: queries, + }) + .then((rsp: any) => { + return { data: gelResponseToDataFrames(rsp) } as DataQueryResponse; + }); + return from(req); + } + + testDatasource() { + return Promise.resolve({}); + } + + newQuery(): ExpressionQuery { + return { + refId: '--', // Replaced with query + type: GELQueryType.math, + datasource: ExpressionDatasourceID, + }; + } +} + +export const ExpressionDatasourceID = '__expr__'; +export const expressionDatasource = new ExpressionDatasourceApi({ + id: -100, + name: ExpressionDatasourceID, +} as DataSourceInstanceSettings); +expressionDatasource.meta = { + id: ExpressionDatasourceID, +} as DataSourcePluginMeta; +expressionDatasource.components = { + QueryEditor: ExpressionQueryEditor, +}; diff --git a/public/app/features/expressions/ExpressionQueryEditor.tsx b/public/app/features/expressions/ExpressionQueryEditor.tsx new file mode 100644 index 00000000000..fb4f8c7064e --- /dev/null +++ b/public/app/features/expressions/ExpressionQueryEditor.tsx @@ -0,0 +1,159 @@ +// Libraries +import React, { PureComponent, ChangeEvent } from 'react'; + +import { FormLabel, QueryEditorProps, Select, FormField } from '@grafana/ui'; +import { SelectableValue, ReducerID } from '@grafana/data'; + +// Types +import { ExpressionQuery, GELQueryType } from './types'; +import { ExpressionDatasourceApi } from './ExpressionDatasource'; + +type Props = QueryEditorProps; + +interface State {} + +const gelTypes: Array> = [ + { value: GELQueryType.math, label: 'Math' }, + { value: GELQueryType.reduce, label: 'Reduce' }, + { value: GELQueryType.resample, label: 'Resample' }, +]; + +const reducerTypes: Array> = [ + { value: ReducerID.min, label: 'Min', description: 'Get the minimum value' }, + { value: ReducerID.max, label: 'Max', description: 'Get the maximum value' }, + { value: ReducerID.mean, label: 'Mean', description: 'Get the average value' }, + { value: ReducerID.sum, label: 'Sum', description: 'Get the sum of all values' }, + { value: ReducerID.count, label: 'Count', description: 'Get the number of values' }, +]; + +const downsamplingTypes: Array> = [ + { value: ReducerID.min, label: 'Min', description: 'Fill with the minimum value' }, + { value: ReducerID.max, label: 'Max', description: 'Fill with the maximum value' }, + { value: ReducerID.mean, label: 'Mean', description: 'Fill with the average value' }, + { value: ReducerID.sum, label: 'Sum', description: 'Fill with the sum of all values' }, +]; + +const upsamplingTypes: Array> = [ + { value: 'pad', label: 'pad', description: 'fill with the last known value' }, + { value: 'backfilling', label: 'backfilling', description: 'fill with the next known value' }, + { value: 'fillna', label: 'fillna', description: 'Fill with NaNs' }, +]; + +export class ExpressionQueryEditor extends PureComponent { + state = {}; + + onSelectGELType = (item: SelectableValue) => { + const { query, onChange } = this.props; + const q = { + ...query, + type: item.value!, + }; + + if (q.type === GELQueryType.reduce) { + if (!q.reducer) { + q.reducer = ReducerID.mean; + } + q.expression = undefined; + } else if (q.type === GELQueryType.resample) { + if (!q.downsampler) { + q.downsampler = ReducerID.mean; + } + if (!q.upsampler) { + q.upsampler = 'fillna'; + } + q.reducer = undefined; + } else { + q.reducer = undefined; + } + + onChange(q); + }; + + onSelectReducer = (item: SelectableValue) => { + const { query, onChange } = this.props; + onChange({ + ...query, + reducer: item.value!, + }); + }; + + onSelectUpsampler = (item: SelectableValue) => { + const { query, onChange } = this.props; + onChange({ + ...query, + upsampler: item.value!, + }); + }; + + onSelectDownsampler = (item: SelectableValue) => { + const { query, onChange } = this.props; + onChange({ + ...query, + downsampler: item.value!, + }); + }; + + onRuleReducer = (item: SelectableValue) => { + const { query, onChange } = this.props; + onChange({ + ...query, + rule: item.value!, + }); + }; + + onExpressionChange = (evt: ChangeEvent) => { + const { query, onChange } = this.props; + onChange({ + ...query, + expression: evt.target.value, + }); + }; + + onRuleChange = (evt: ChangeEvent) => { + const { query, onChange } = this.props; + onChange({ + ...query, + rule: evt.target.value, + }); + }; + + render() { + const { query } = this.props; + const selected = gelTypes.find(o => o.value === query.type); + const reducer = reducerTypes.find(o => o.value === query.reducer); + const downsampler = downsamplingTypes.find(o => o.value === query.downsampler); + const upsampler = upsamplingTypes.find(o => o.value === query.upsampler); + + return ( +
+
+ + + + )} +
+ {query.type === GELQueryType.math && ( +