transform: add expressions to query editor (w/ feature flag) (#20072)

for use with gel which is not released yet.
This commit is contained in:
Ryan McKinley
2019-10-30 11:38:28 -07:00
committed by Kyle Brandt
parent 2bb4684741
commit 861eb72113
16 changed files with 726 additions and 126 deletions

View File

@@ -29,6 +29,7 @@ import { PluginHelp } from 'app/core/components/PluginHelp/PluginHelp';
import { addQuery } from 'app/core/utils/query';
import { Unsubscribable } from 'rxjs';
import { isSharedDashboardQuery, DashboardQueryEditor } from 'app/plugins/datasource/dashboard';
import { expressionDatasource, ExpressionDatasourceID } from 'app/features/expressions/ExpressionDatasource';
interface Props {
panel: PanelModel;
@@ -97,9 +98,11 @@ export class QueriesTab extends PureComponent<Props, State> {
if (datasource.meta.mixed) {
// Set the datasource on all targets
panel.targets.forEach(target => {
target.datasource = panel.datasource;
if (!target.datasource) {
target.datasource = config.defaultDatasource;
if (target.datasource !== ExpressionDatasourceID) {
target.datasource = panel.datasource;
if (!target.datasource) {
target.datasource = config.defaultDatasource;
}
}
});
} else if (currentDS) {
@@ -107,7 +110,9 @@ export class QueriesTab extends PureComponent<Props, State> {
if (currentDS.meta.mixed) {
// Remove the explicit datasource
for (const target of panel.targets) {
delete target.datasource;
if (target.datasource !== ExpressionDatasourceID) {
delete target.datasource;
}
}
} else if (currentDS.meta.id !== datasource.meta.id) {
// we are changing data source type, clear queries
@@ -150,6 +155,11 @@ export class QueriesTab extends PureComponent<Props, State> {
this.onScrollBottom();
};
onAddExpressionClick = () => {
this.onUpdateQueries(addQuery(this.props.panel.targets, expressionDatasource.newQuery()));
this.onScrollBottom();
};
onScrollBottom = () => {
this.setState({ scrollTop: this.state.scrollTop + 10000 });
};
@@ -168,6 +178,11 @@ export class QueriesTab extends PureComponent<Props, State> {
</button>
)}
{isAddingMixed && this.renderMixedPicker()}
{config.featureToggles.expressions && (
<button className="btn navbar-button" onClick={this.onAddExpressionClick}>
Add Expression
</button>
)}
</>
);
};

View File

@@ -14,6 +14,7 @@ import {
DataQueryError,
} from '@grafana/ui';
import { LoadingState, dateMath, toDataFrame, DataFrame, guessFieldTypes } from '@grafana/data';
import { ExpressionDatasourceID, expressionDatasource } from 'app/features/expressions/ExpressionDatasource';
type MapOfResponsePackets = { [str: string]: DataQueryResponse };
@@ -132,6 +133,16 @@ function cancelNetworkRequestsOnUnsubscribe(req: DataQueryRequest) {
}
export function callQueryMethod(datasource: DataSourceApi, request: DataQueryRequest) {
console.log('CALL', request.targets);
// If any query has an expression, use the expression endpoint
for (const target of request.targets) {
if (target.datasource === ExpressionDatasourceID) {
return expressionDatasource.query(request);
}
}
// Otherwise it is a standard datasource request
const returnVal = datasource.query(request);
return from(returnVal);
}

View File

@@ -0,0 +1,84 @@
import {
DataSourceApi,
DataQueryRequest,
DataQueryResponse,
DataSourceInstanceSettings,
DataSourcePluginMeta,
} from '@grafana/ui';
import { ExpressionQuery, GELQueryType } from './types';
import { ExpressionQueryEditor } from './ExpressionQueryEditor';
import { Observable, from } from 'rxjs';
import { config } from '@grafana/runtime';
import { getBackendSrv } from 'app/core/services/backend_srv';
import { gelResponseToDataFrames } from './util';
/**
* This is a singleton that is not actually instanciated
*/
export class ExpressionDatasourceApi extends DataSourceApi<ExpressionQuery> {
constructor(instanceSettings: DataSourceInstanceSettings) {
super(instanceSettings);
}
getCollapsedText(query: ExpressionQuery) {
return `Expression: ${query.type}`;
}
query(request: DataQueryRequest): Observable<DataQueryResponse> {
const { targets, intervalMs, maxDataPoints, range } = request;
const orgId = (window as any).grafanaBootData.user.orgId;
const queries = targets.map(q => {
if (q.datasource === ExpressionDatasourceID) {
return {
...q,
datasourceId: this.id,
orgId,
};
}
const ds = config.datasources[q.datasource || config.defaultDatasource];
return {
...q,
datasourceId: ds.id,
intervalMs,
maxDataPoints,
orgId,
// ?? alias: templateSrv.replace(q.alias || ''),
};
});
const req: Promise<DataQueryResponse> = getBackendSrv()
.post('/api/tsdb/query/v2', {
from: range.from.valueOf().toString(),
to: range.to.valueOf().toString(),
queries: queries,
})
.then((rsp: any) => {
return { data: gelResponseToDataFrames(rsp) } as DataQueryResponse;
});
return from(req);
}
testDatasource() {
return Promise.resolve({});
}
newQuery(): ExpressionQuery {
return {
refId: '--', // Replaced with query
type: GELQueryType.math,
datasource: ExpressionDatasourceID,
};
}
}
export const ExpressionDatasourceID = '__expr__';
export const expressionDatasource = new ExpressionDatasourceApi({
id: -100,
name: ExpressionDatasourceID,
} as DataSourceInstanceSettings);
expressionDatasource.meta = {
id: ExpressionDatasourceID,
} as DataSourcePluginMeta;
expressionDatasource.components = {
QueryEditor: ExpressionQueryEditor,
};

View File

@@ -0,0 +1,159 @@
// Libraries
import React, { PureComponent, ChangeEvent } from 'react';
import { FormLabel, QueryEditorProps, Select, FormField } from '@grafana/ui';
import { SelectableValue, ReducerID } from '@grafana/data';
// Types
import { ExpressionQuery, GELQueryType } from './types';
import { ExpressionDatasourceApi } from './ExpressionDatasource';
type Props = QueryEditorProps<ExpressionDatasourceApi, ExpressionQuery>;
interface State {}
const gelTypes: Array<SelectableValue<GELQueryType>> = [
{ value: GELQueryType.math, label: 'Math' },
{ value: GELQueryType.reduce, label: 'Reduce' },
{ value: GELQueryType.resample, label: 'Resample' },
];
const reducerTypes: Array<SelectableValue<string>> = [
{ value: ReducerID.min, label: 'Min', description: 'Get the minimum value' },
{ value: ReducerID.max, label: 'Max', description: 'Get the maximum value' },
{ value: ReducerID.mean, label: 'Mean', description: 'Get the average value' },
{ value: ReducerID.sum, label: 'Sum', description: 'Get the sum of all values' },
{ value: ReducerID.count, label: 'Count', description: 'Get the number of values' },
];
const downsamplingTypes: Array<SelectableValue<string>> = [
{ value: ReducerID.min, label: 'Min', description: 'Fill with the minimum value' },
{ value: ReducerID.max, label: 'Max', description: 'Fill with the maximum value' },
{ value: ReducerID.mean, label: 'Mean', description: 'Fill with the average value' },
{ value: ReducerID.sum, label: 'Sum', description: 'Fill with the sum of all values' },
];
const upsamplingTypes: Array<SelectableValue<string>> = [
{ value: 'pad', label: 'pad', description: 'fill with the last known value' },
{ value: 'backfilling', label: 'backfilling', description: 'fill with the next known value' },
{ value: 'fillna', label: 'fillna', description: 'Fill with NaNs' },
];
export class ExpressionQueryEditor extends PureComponent<Props, State> {
state = {};
onSelectGELType = (item: SelectableValue<GELQueryType>) => {
const { query, onChange } = this.props;
const q = {
...query,
type: item.value!,
};
if (q.type === GELQueryType.reduce) {
if (!q.reducer) {
q.reducer = ReducerID.mean;
}
q.expression = undefined;
} else if (q.type === GELQueryType.resample) {
if (!q.downsampler) {
q.downsampler = ReducerID.mean;
}
if (!q.upsampler) {
q.upsampler = 'fillna';
}
q.reducer = undefined;
} else {
q.reducer = undefined;
}
onChange(q);
};
onSelectReducer = (item: SelectableValue<string>) => {
const { query, onChange } = this.props;
onChange({
...query,
reducer: item.value!,
});
};
onSelectUpsampler = (item: SelectableValue<string>) => {
const { query, onChange } = this.props;
onChange({
...query,
upsampler: item.value!,
});
};
onSelectDownsampler = (item: SelectableValue<string>) => {
const { query, onChange } = this.props;
onChange({
...query,
downsampler: item.value!,
});
};
onRuleReducer = (item: SelectableValue<string>) => {
const { query, onChange } = this.props;
onChange({
...query,
rule: item.value!,
});
};
onExpressionChange = (evt: ChangeEvent<any>) => {
const { query, onChange } = this.props;
onChange({
...query,
expression: evt.target.value,
});
};
onRuleChange = (evt: ChangeEvent<any>) => {
const { query, onChange } = this.props;
onChange({
...query,
rule: evt.target.value,
});
};
render() {
const { query } = this.props;
const selected = gelTypes.find(o => o.value === query.type);
const reducer = reducerTypes.find(o => o.value === query.reducer);
const downsampler = downsamplingTypes.find(o => o.value === query.downsampler);
const upsampler = upsamplingTypes.find(o => o.value === query.upsampler);
return (
<div>
<div className="form-field">
<Select options={gelTypes} value={selected} onChange={this.onSelectGELType} />
{query.type === GELQueryType.reduce && (
<>
<FormLabel width={5}>Function:</FormLabel>
<Select options={reducerTypes} value={reducer} onChange={this.onSelectReducer} />
<FormField label="Fields:" labelWidth={5} onChange={this.onExpressionChange} value={query.expression} />
</>
)}
</div>
{query.type === GELQueryType.math && (
<textarea value={query.expression} onChange={this.onExpressionChange} className="gf-form-input" rows={2} />
)}
{query.type === GELQueryType.resample && (
<>
<div>
<FormField label="Series:" labelWidth={5} onChange={this.onExpressionChange} value={query.expression} />
<FormField label="Rule:" labelWidth={5} onChange={this.onRuleChange} value={query.rule} />
</div>
<div>
<FormLabel width={12}>Downsample Function:</FormLabel>
<Select options={downsamplingTypes} value={downsampler} onChange={this.onSelectDownsampler} />
<FormLabel width={12}>Upsample Function:</FormLabel>
<Select options={upsamplingTypes} value={upsampler} onChange={this.onSelectUpsampler} />
</div>
</>
)}
</div>
);
}
}

View File

@@ -0,0 +1,128 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`GEL Utils should parse output with dataframe 1`] = `
Array [
Object {
"fields": Array [
Object {
"config": Object {},
"name": "Time",
"type": "time",
"values": Int32Array [
882710016,
365389179,
1587742720,
365389180,
-2002191872,
365389181,
-1297159168,
365389182,
-592126464,
365389183,
112906240,
365389185,
817938944,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
],
},
Object {
"config": Object {},
"name": "",
"type": "number",
"values": Float64Array [
3,
3,
3,
5,
5,
5,
3,
3,
3,
5,
5,
5,
3,
],
},
],
"labels": undefined,
"meta": undefined,
"name": undefined,
"refId": undefined,
},
Object {
"fields": Array [
Object {
"config": Object {},
"name": "Time",
"type": "time",
"values": Int32Array [
882710016,
365389179,
1587742720,
365389180,
-2002191872,
365389181,
-1297159168,
365389182,
-592126464,
365389183,
112906240,
365389185,
817938944,
365389186,
1522971648,
365389187,
-2066962944,
365389188,
-1361930240,
365389189,
-656897536,
365389190,
48135168,
365389192,
753167872,
365389193,
],
},
Object {
"config": Object {},
"name": "GB-series",
"type": "number",
"values": Float64Array [
0,
0,
0,
2,
2,
2,
0,
0,
0,
2,
2,
2,
0,
],
},
],
"labels": undefined,
"meta": undefined,
"name": undefined,
"refId": undefined,
},
]
`;

View File

@@ -0,0 +1,20 @@
import { DataQuery } from '@grafana/ui';
export enum GELQueryType {
math = 'math',
reduce = 'reduce',
resample = 'resample',
}
/**
* For now this is a single object to cover all the types.... would likely
* want to split this up by type as the complexity increases
*/
export interface ExpressionQuery extends DataQuery {
type: GELQueryType;
reducer?: string;
expression?: string;
rule?: string;
downsampler?: string;
upsampler?: string;
}

View File

@@ -0,0 +1,48 @@
import { gelResponseToDataFrames } from './util';
import { toDataFrameDTO } from '@grafana/data';
/* tslint:disable */
const resp = {
results: {
'': {
refId: '',
dataframes: [
'QVJST1cxAACsAQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAOD+//8IAAAADAAAAAIAAABHQwAABQAAAHJlZklkAAAAAP///wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAAlAAAAAQAAACG////FAAAAGAAAABgAAAAAAADAWAAAAACAAAALAAAAAQAAABQ////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAAB0////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAAAAABm////AAACAAAAAAAAABIAGAAUABMAEgAMAAAACAAEABIAAAAUAAAAbAAAAHQAAAAAAAoBdAAAAAIAAAA0AAAABAAAANz///8IAAAAEAAAAAQAAAB0aW1lAAAAAAQAAAB0eXBlAAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAFRpbWUAAAAABAAAAG5hbWUAAAAAAAAAAAAABgAIAAYABgAAAAAAAwAEAAAAVGltZQAAAAC8AAAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAA0AAAAAAAAAAUAAAAAAAAAwMACgAYAAwACAAEAAoAAAAUAAAAWAAAAA0AAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABoAAAAAAAAAGgAAAAAAAAAAAAAAAAAAABoAAAAAAAAAGgAAAAAAAAAAAAAAAIAAAANAAAAAAAAAAAAAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAFp00e2XHFQAIo158ZccVAPqoiH1lxxUA7K6yfmXHFQDetNx/ZccVANC6BoFlxxUAwsAwgmXHFQC0xlqDZccVAKbMhIRlxxUAmNKuhWXHFQCK2NiGZccVAHzeAohlxxUAbuQsiWXHFQAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAUQAAAAAAAAAhAAAAAAAAACEAAAAAAAAAIQAAAAAAAABRAAAAAAAAAFEAAAAAAAAAUQAAAAAAAAAhAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADgAAAAAAAMAAQAAALgBAAAAAAAAwAAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAKAAwAAAAIAAQACgAAAAgAAABQAAAAAgAAACgAAAAEAAAA4P7//wgAAAAMAAAAAgAAAEdDAAAFAAAAcmVmSWQAAAAA////CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAAIAAACUAAAABAAAAIb///8UAAAAYAAAAGAAAAAAAAMBYAAAAAIAAAAsAAAABAAAAFD///8IAAAAEAAAAAYAAABudW1iZXIAAAQAAAB0eXBlAAAAAHT///8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAAAAAAGb///8AAAIAAAAAAAAAEgAYABQAEwASAAwAAAAIAAQAEgAAABQAAABsAAAAdAAAAAAACgF0AAAAAgAAADQAAAAEAAAA3P///wgAAAAQAAAABAAAAHRpbWUAAAAABAAAAHR5cGUAAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAVGltZQAAAAAEAAAAbmFtZQAAAAAAAAAAAAAGAAgABgAGAAAAAAADAAQAAABUaW1lAAAAANgBAABBUlJPVzE=',
'QVJST1cxAAC8AQAAEAAAAAAACgAOAAwACwAEAAoAAAAUAAAAAAAAAQMACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAND+//8IAAAADAAAAAIAAABHQgAABQAAAHJlZklkAAAA8P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAApAAAAAQAAAB2////FAAAAGgAAABoAAAAAAADAWgAAAACAAAALAAAAAQAAABA////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAABk////CAAAABQAAAAJAAAAR0Itc2VyaWVzAAAABAAAAG5hbWUAAAAAAAAAAF7///8AAAIACQAAAEdCLXNlcmllcwASABgAFAATABIADAAAAAgABAASAAAAFAAAAGwAAAB0AAAAAAAKAXQAAAACAAAANAAAAAQAAADc////CAAAABAAAAAEAAAAdGltZQAAAAAEAAAAdHlwZQAAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAAvAAAABQAAAAAAAAADAAWABQAEwAMAAQADAAAANAAAAAAAAAAFAAAAAAAAAMDAAoAGAAMAAgABAAKAAAAFAAAAFgAAAANAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaAAAAAAAAABoAAAAAAAAAAAAAAAAAAAAaAAAAAAAAABoAAAAAAAAAAAAAAACAAAADQAAAAAAAAAAAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAAAAABadNHtlxxUACKNefGXHFQD6qIh9ZccVAOyusn5lxxUA3rTcf2XHFQDQugaBZccVAMLAMIJlxxUAtMZag2XHFQCmzISEZccVAJjSroVlxxUAitjYhmXHFQB83gKIZccVAG7kLIllxxUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAEAAAAAAAAAAABAAAAAMABQAEgAMAAgABAAMAAAAEAAAACwAAAA4AAAAAAADAAEAAADIAQAAAAAAAMAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAUAAAAAIAAAAoAAAABAAAAND+//8IAAAADAAAAAIAAABHQgAABQAAAHJlZklkAAAA8P7//wgAAAAMAAAAAAAAAAAAAAAEAAAAbmFtZQAAAAACAAAApAAAAAQAAAB2////FAAAAGgAAABoAAAAAAADAWgAAAACAAAALAAAAAQAAABA////CAAAABAAAAAGAAAAbnVtYmVyAAAEAAAAdHlwZQAAAABk////CAAAABQAAAAJAAAAR0Itc2VyaWVzAAAABAAAAG5hbWUAAAAAAAAAAF7///8AAAIACQAAAEdCLXNlcmllcwASABgAFAATABIADAAAAAgABAASAAAAFAAAAGwAAAB0AAAAAAAKAXQAAAACAAAANAAAAAQAAADc////CAAAABAAAAAEAAAAdGltZQAAAAAEAAAAdHlwZQAAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABUaW1lAAAAAAQAAABuYW1lAAAAAAAAAAAAAAYACAAGAAYAAAAAAAMABAAAAFRpbWUAAAAA6AEAAEFSUk9XMQ==',
],
series: [] as any[],
tables: null as any,
frames: null as any,
},
},
};
/* tslint:enable */
describe('GEL Utils', () => {
// test('should parse sample GEL output', () => {
// const frames = gelResponseToDataFrames(resp);
// const frame = frames[0];
// expect(frame.name).toEqual('BBB');
// expect(frame.fields.length).toEqual(2);
// expect(frame.length).toEqual(resp.Frames[0].fields[0].values.length);
// const timeField = frame.fields[0];
// expect(timeField.name).toEqual('Time');
// // The whole response
// expect(frames).toMatchSnapshot();
// });
test('should parse output with dataframe', () => {
const frames = gelResponseToDataFrames(resp);
for (const frame of frames) {
console.log('Frame', frame.refId + ' // ' + frame.labels);
for (const field of frame.fields) {
console.log(' > ', field.name, field.values.toArray());
}
}
const norm = frames.map(f => toDataFrameDTO(f));
expect(norm).toMatchSnapshot();
});
});

View File

@@ -0,0 +1,62 @@
import { DataFrame, FieldType, Field, Vector } from '@grafana/data';
import { Table, ArrowType } from 'apache-arrow';
export function base64StringToArrowTable(text: string) {
const b64 = atob(text);
const arr = Uint8Array.from(b64, c => {
return c.charCodeAt(0);
});
return Table.from(arr);
}
export function arrowTableToDataFrame(table: Table): DataFrame {
const fields: Field[] = [];
for (let i = 0; i < table.numCols; i++) {
const col = table.getColumnAt(i);
if (col) {
const schema = table.schema.fields[i];
let type = FieldType.other;
const values: Vector<any> = col;
switch ((schema.typeId as unknown) as ArrowType) {
case ArrowType.Decimal:
case ArrowType.Int:
case ArrowType.FloatingPoint: {
type = FieldType.number;
break;
}
case ArrowType.Bool: {
type = FieldType.boolean;
break;
}
case ArrowType.Timestamp: {
type = FieldType.time;
break;
}
default:
console.log('UNKNOWN Type:', schema);
}
fields.push({
name: col.name,
type,
config: {}, // TODO, pull from metadata
values,
});
}
}
return {
fields,
length: table.length,
};
}
export function gelResponseToDataFrames(rsp: any): DataFrame[] {
const frames: DataFrame[] = [];
for (const res of Object.values(rsp.results)) {
for (const b of (res as any).dataframes) {
const t = base64StringToArrowTable(b as string);
frames.push(arrowTableToDataFrame(t));
}
}
return frames;
}

View File

@@ -14,6 +14,9 @@ import { auto } from 'angular';
import { TemplateSrv } from '../templating/template_srv';
import { GrafanaRootScope } from 'app/routes/GrafanaCtrl';
// Pretend Datasource
import { expressionDatasource } from 'app/features/expressions/ExpressionDatasource';
export class DatasourceSrv implements DataSourceService {
datasources: { [name: string]: DataSourceApi };
@@ -56,6 +59,12 @@ export class DatasourceSrv implements DataSourceService {
}
loadDatasource(name: string): Promise<DataSourceApi> {
// Expression Datasource (not a real datasource)
if (name === expressionDatasource.name) {
this.datasources[name] = expressionDatasource;
return this.$q.when(expressionDatasource);
}
const dsConfig = config.datasources[name];
if (!dsConfig) {
return this.$q.reject({ message: 'Datasource named ' + name + ' was not found' });