IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Transformations: Use Observable ()

Browse Source 
* WIP: changes transformer API to Observable

* Refactor: changes ResultProcessor

* WIP: TransformationUI

* wip

* Refactor: Cleaning up code in CalculateFieldTransformerEditorProps

* Refactor: pushing editor and input output down to TransformationEditor

* Refactor: renaming props

* Refactor: makes transformDataFrame more readable

* Refactor: fixes editor issues

* Refactor: changes after merge with master and first tests

* Tests: completes the tests for the Explore changes

* Tests: fixes all transform tests

* Tests: fixed annotations test

* Tests: fixed typechecks

* Refactor: changes transform interface to MonoTypeOperatorFunction

* Chore: cleans up typings

* Update packages/grafana-data/src/transformations/transformDataFrame.ts

Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>

* Tests: fixes broken tests

Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>
This commit is contained in:
Hugo Häggmark 2020-10-06 07:55:09 +02:00 committed by GitHub
parent 0bf67612d1
commit de1b2bdc4c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
59 changed files with 3540 additions and 2686 deletions

71
packages/grafana-data/src/transformations/transformDataFrame.ts
View File

@ -1,37 +1,68 @@
import { MonoTypeOperatorFunction, Observable, of } from 'rxjs';
import { map, mergeMap } from 'rxjs/operators';
import { DataFrame, DataTransformerConfig } from '../types';
import { standardTransformersRegistry } from './standardTransformersRegistry';
import { standardTransformersRegistry, TransformerRegistyItem } from './standardTransformersRegistry';
/**
* Apply configured transformations to the input data
*/
export function transformDataFrame(options: DataTransformerConfig[], data: DataFrame[]): DataFrame[] {
let processed = data;
for (const config of options) {
const info = standardTransformersRegistry.get(config.id);
const getOperator = (config: DataTransformerConfig): MonoTypeOperatorFunction<DataFrame[]> => source => {
const info = standardTransformersRegistry.get(config.id);
if (!info) {
return data;
}
if (!info) {
return source;
}
const defaultOptions = info.transformation.defaultOptions ?? {};
const options = { ...defaultOptions, ...config.options };
const transformer = info.transformation.transformer(options);
const after = transformer(processed);
const defaultOptions = info.transformation.defaultOptions ?? {};
const options = { ...defaultOptions, ...config.options };
// Add a key to the metadata if the data changed
if (after && after !== processed) {
return source.pipe(
mergeMap(before => of(before).pipe(info.transformation.operator(options), postProcessTransform(before, info)))
);
};
const postProcessTransform = (
before: DataFrame[],
info: TransformerRegistyItem<any>
): MonoTypeOperatorFunction<DataFrame[]> => source =>
source.pipe(
map(after => {
if (after === before) {
return after;
}
// Add a key to the metadata if the data changed
for (const series of after) {
if (!series.meta) {
series.meta = {};
}
if (!series.meta.transformations) {
series.meta.transformations = [info.id];
} else {
series.meta.transformations = [...series.meta.transformations, info.id];
}
}
processed = after;
}
return after;
})
);
/**
* Apply configured transformations to the input data
*/
export function transformDataFrame(options: DataTransformerConfig[], data: DataFrame[]): Observable<DataFrame[]> {
const stream = of<DataFrame[]>(data);
if (!options.length) {
return stream;
}
return processed;
const operators: Array<MonoTypeOperatorFunction<DataFrame[]>> = [];
for (let index = 0; index < options.length; index++) {
const config = options[index];
operators.push(getOperator(config));
}
// @ts-ignore TypeScript has a hard time understanding this construct
return stream.pipe.apply(stream, operators);
}

26
packages/grafana-data/src/transformations/transformers/append.test.ts
View File

@ -3,6 +3,7 @@ import { toDataFrame } from '../../dataframe/processDataFrame';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { appendTransformer } from './append';
import { transformDataFrame } from '../transformDataFrame';
import { observableTester } from '../../utils/tests/observableTester';
const seriesAB = toDataFrame({
columns: [{ text: 'A' }, { text: 'B' }],
@ -24,21 +25,28 @@ describe('Append Transformer', () => {
beforeAll(() => {
mockTransformationsRegistry([appendTransformer]);
});
it('filters by include', () => {
it('filters by include', done => {
const cfg = {
id: DataTransformerID.append,
options: {},
};
const processed = transformDataFrame([cfg], [seriesAB, seriesBC])[0];
expect(processed.fields.length).toBe(3);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesAB, seriesBC]),
expect: data => {
const processed = data[0];
expect(processed.fields.length).toBe(3);
const fieldA = processed.fields[0];
const fieldB = processed.fields[1];
const fieldC = processed.fields[2];
const fieldA = processed.fields[0];
const fieldB = processed.fields[1];
const fieldC = processed.fields[2];
expect(fieldA.values.toArray()).toEqual([1, 2, 3, 4]);
expect(fieldB.values.toArray()).toEqual([100, 200, null, null]);
expect(fieldC.values.toArray()).toEqual([null, null, 3000, 4000]);
expect(fieldA.values.toArray()).toEqual([1, 2, 3, 4]);
expect(fieldB.values.toArray()).toEqual([100, 200, null, null]);
expect(fieldC.values.toArray()).toEqual([null, null, 3000, 4000]);
},
done,
});
});
});

84
packages/grafana-data/src/transformations/transformers/append.ts
View File

@ -1,4 +1,5 @@
import { DataFrame } from '../../types/dataFrame';
import { map } from 'rxjs/operators';
import { DataTransformerID } from './ids';
import { MutableDataFrame } from '../../dataframe/MutableDataFrame';
import { DataTransformerInfo } from '../../types/transformations';
@ -15,45 +16,46 @@ export const appendTransformer: DataTransformerInfo<AppendOptions> = {
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: AppendOptions) => {
return (data: DataFrame[]) => {
if (data.length < 2) {
return data;
}
// Add the first row
const processed = new MutableDataFrame();
for (const f of data[0].fields) {
processed.addField({
...f,
values: [...f.values.toArray()],
});
}
for (let i = 1; i < data.length; i++) {
const frame = data[i];
const startLength = frame.length;
for (let j = 0; j < frame.fields.length; j++) {
const src = frame.fields[j];
let vals = processed.values[src.name];
if (!vals) {
vals = processed.addField(
{
...src,
values: [],
},
startLength
).values;
}
// Add each row
for (let k = 0; k < frame.length; k++) {
vals.add(src.values.get(k));
}
operator: options => source =>
source.pipe(
map(data => {
if (data.length < 2) {
return data;
}
processed.validate();
}
return [processed];
};
},
// Add the first row
const processed = new MutableDataFrame();
for (const f of data[0].fields) {
processed.addField({
...f,
values: [...f.values.toArray()],
});
}
for (let i = 1; i < data.length; i++) {
const frame = data[i];
const startLength = frame.length;
for (let j = 0; j < frame.fields.length; j++) {
const src = frame.fields[j];
let vals = processed.values[src.name];
if (!vals) {
vals = processed.addField(
{
...src,
values: [],
},
startLength
).values;
}
// Add each row
for (let k = 0; k < frame.length; k++) {
vals.add(src.values.get(k));
}
}
processed.validate();
}
return [processed];
})
),
};

189
packages/grafana-data/src/transformations/transformers/calculateField.test.ts
View File

@ -4,9 +4,10 @@ import { FieldType } from '../../types/dataFrame';
import { ReducerID } from '../fieldReducer';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { transformDataFrame } from '../transformDataFrame';
import { calculateFieldTransformer, CalculateFieldMode, ReduceOptions } from './calculateField';
import { CalculateFieldMode, calculateFieldTransformer, ReduceOptions } from './calculateField';
import { DataFrameView } from '../../dataframe';
import { BinaryOperationID } from '../../utils';
import { observableTester } from '../../utils/tests/observableTester';
const seriesA = toDataFrame({
fields: [
@ -29,7 +30,7 @@ describe('calculateField transformer w/ timeseries', () => {
mockTransformationsRegistry([calculateFieldTransformer]);
});
it('will filter and alias', () => {
it('will filter and alias', done => {
const cfg = {
id: DataTransformerID.calculateField,
options: {
@ -38,31 +39,35 @@ describe('calculateField transformer w/ timeseries', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesA, seriesBC])[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toMatchInlineSnapshot(`
Array [
Object {
"A": 1,
"B": 2,
"C": 3,
"D": "first",
"The Total": 6,
"TheTime": 1000,
},
Object {
"A": 100,
"B": 200,
"C": 300,
"D": "second",
"The Total": 600,
"TheTime": 2000,
},
]
`);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesA, seriesBC]),
expect: data => {
const filtered = data[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toEqual([
{
A: 1,
B: 2,
C: 3,
D: 'first',
'The Total': 6,
TheTime: 1000,
},
{
A: 100,
B: 200,
C: 300,
D: 'second',
'The Total': 600,
TheTime: 2000,
},
]);
},
done,
});
});
it('will replace other fields', () => {
it('will replace other fields', done => {
const cfg = {
id: DataTransformerID.calculateField,
options: {
@ -74,23 +79,27 @@ describe('calculateField transformer w/ timeseries', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesA, seriesBC])[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toMatchInlineSnapshot(`
Array [
Object {
"Mean": 2,
"TheTime": 1000,
},
Object {
"Mean": 200,
"TheTime": 2000,
},
]
`);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesA, seriesBC]),
expect: data => {
const filtered = data[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toEqual([
{
Mean: 2,
TheTime: 1000,
},
{
Mean: 200,
TheTime: 2000,
},
]);
},
done,
});
});
it('will filter by name', () => {
it('will filter by name', done => {
const cfg = {
id: DataTransformerID.calculateField,
options: {
@ -103,23 +112,27 @@ describe('calculateField transformer w/ timeseries', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesBC])[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toMatchInlineSnapshot(`
Array [
Object {
"Mean": 2,
"TheTime": 1000,
},
Object {
"Mean": 200,
"TheTime": 2000,
},
]
`);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesBC]),
expect: data => {
const filtered = data[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toEqual([
{
Mean: 2,
TheTime: 1000,
},
{
Mean: 200,
TheTime: 2000,
},
]);
},
done,
});
});
it('binary math', () => {
it('binary math', done => {
const cfg = {
id: DataTransformerID.calculateField,
options: {
@ -133,23 +146,27 @@ describe('calculateField transformer w/ timeseries', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesBC])[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toMatchInlineSnapshot(`
Array [
Object {
"B + C": 5,
"TheTime": 1000,
},
Object {
"B + C": 500,
"TheTime": 2000,
},
]
`);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesBC]),
expect: data => {
const filtered = data[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toEqual([
{
'B + C': 5,
TheTime: 1000,
},
{
'B + C': 500,
TheTime: 2000,
},
]);
},
done,
});
});
it('field + static number', () => {
it('field + static number', done => {
const cfg = {
id: DataTransformerID.calculateField,
options: {
@ -163,19 +180,23 @@ describe('calculateField transformer w/ timeseries', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesBC])[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toMatchInlineSnapshot(`
Array [
Object {
"B + 2": 4,
"TheTime": 1000,
},
Object {
"B + 2": 202,
"TheTime": 2000,
},
]
`);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesBC]),
expect: data => {
const filtered = data[0];
const rows = new DataFrameView(filtered).toArray();
expect(rows).toEqual([
{
'B + 2': 4,
TheTime: 1000,
},
{
'B + 2': 202,
TheTime: 2000,
},
]);
},
done,
});
});
});

106
packages/grafana-data/src/transformations/transformers/calculateField.ts
View File

@ -1,16 +1,18 @@
import { DataFrame, DataTransformerInfo, Vector, FieldType, Field, NullValueMode } from '../../types';
import { map } from 'rxjs/operators';
import { DataFrame, DataTransformerInfo, Field, FieldType, NullValueMode, Vector } from '../../types';
import { DataTransformerID } from './ids';
import { ReducerID, fieldReducers } from '../fieldReducer';
import { doStandardCalcs, fieldReducers, ReducerID } from '../fieldReducer';
import { getFieldMatcher } from '../matchers';
import { FieldMatcherID } from '../matchers/ids';
import { RowVector } from '../../vector/RowVector';
import { ArrayVector, BinaryOperationVector, ConstantVector } from '../../vector';
import { doStandardCalcs } from '../fieldReducer';
import { getTimeField } from '../../dataframe/processDataFrame';
import defaults from 'lodash/defaults';
import { BinaryOperationID, binaryOperators } from '../../utils/binaryOperators';
import { ensureColumnsTransformer } from './ensureColumns';
import { getFieldDisplayName } from '../../field';
import { noopTransformer } from './noop';
export enum CalculateFieldMode {
ReduceRow = 'reduceRow',
@ -68,56 +70,60 @@ export const calculateFieldTransformer: DataTransformerInfo<CalculateFieldTransf
reducer: ReducerID.sum,
},
},
transformer: options => (data: DataFrame[]) => {
if (options && options.timeSeries !== false) {
data = ensureColumnsTransformer.transformer(null)(data);
}
operator: options => outerSource => {
const operator =
options && options.timeSeries !== false ? ensureColumnsTransformer.operator(null) : noopTransformer.operator({});
const mode = options.mode ?? CalculateFieldMode.ReduceRow;
let creator: ValuesCreator | undefined = undefined;
return outerSource.pipe(
operator,
map(data => {
const mode = options.mode ?? CalculateFieldMode.ReduceRow;
let creator: ValuesCreator | undefined = undefined;
if (mode === CalculateFieldMode.ReduceRow) {
creator = getReduceRowCreator(defaults(options.reduce, defaultReduceOptions), data);
} else if (mode === CalculateFieldMode.BinaryOperation) {
creator = getBinaryCreator(defaults(options.binary, defaultBinaryOptions), data);
}
// Nothing configured
if (!creator) {
return data;
}
return data.map(frame => {
// delegate field creation to the specific function
const values = creator!(frame);
if (!values) {
return frame;
}
const field = {
name: getNameFromOptions(options),
type: FieldType.number,
config: {},
values,
};
let fields: Field[] = [];
// Replace all fields with the single field
if (options.replaceFields) {
const { timeField } = getTimeField(frame);
if (timeField && options.timeSeries !== false) {
fields = [timeField, field];
} else {
fields = [field];
if (mode === CalculateFieldMode.ReduceRow) {
creator = getReduceRowCreator(defaults(options.reduce, defaultReduceOptions), data);
} else if (mode === CalculateFieldMode.BinaryOperation) {
creator = getBinaryCreator(defaults(options.binary, defaultBinaryOptions), data);
}
} else {
fields = [...frame.fields, field];
}
return {
...frame,
fields,
};
});
// Nothing configured
if (!creator) {
return data;
}
return data.map(frame => {
// delegate field creation to the specific function
const values = creator!(frame);
if (!values) {
return frame;
}
const field = {
name: getNameFromOptions(options),
type: FieldType.number,
config: {},
values,
};
let fields: Field[] = [];
// Replace all fields with the single field
if (options.replaceFields) {
const { timeField } = getTimeField(frame);
if (timeField && options.timeSeries !== false) {
fields = [timeField, field];
} else {
fields = [field];
}
} else {
fields = [...frame.fields, field];
}
return {
...frame,
fields,
};
});
})
);
},
};

36
packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts
View File

@ -5,6 +5,7 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformatio
import { transformDataFrame } from '../transformDataFrame';
import { ensureColumnsTransformer } from './ensureColumns';
import { seriesToColumnsTransformer } from './seriesToColumns';
import { observableTester } from '../../utils/tests/observableTester';
const seriesA = toDataFrame({
fields: [
@ -35,17 +36,19 @@ describe('ensureColumns transformer', () => {
mockTransformationsRegistry([ensureColumnsTransformer, seriesToColumnsTransformer]);
});
it('will transform to columns if time field exists and multiple frames', () => {
it('will transform to columns if time field exists and multiple frames', done => {
const cfg = {
id: DataTransformerID.ensureColumns,
options: {},
};
const data = [seriesA, seriesBC];
const filtered = transformDataFrame([cfg], data);
expect(filtered.length).toEqual(1);
expect(filtered[0]).toMatchInlineSnapshot(`
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], data),
expect: filtered => {
expect(filtered.length).toEqual(1);
expect(filtered[0]).toMatchInlineSnapshot(`
Object {
"fields": Array [
Object {
@ -108,29 +111,42 @@ describe('ensureColumns transformer', () => {
"refId": undefined,
}
`);
},
done,
});
});
it('will not transform to columns if time field is missing for any of the series', () => {
it('will not transform to columns if time field is missing for any of the series', done => {
const cfg = {
id: DataTransformerID.ensureColumns,
options: {},
};
const data = [seriesBC, seriesNoTime];
const filtered = transformDataFrame([cfg], data);
expect(filtered).toEqual(data);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], data),
expect: filtered => {
expect(filtered).toEqual(data);
},
done,
});
});
it('will not transform to columns if only one series', () => {
it('will not transform to columns if only one series', done => {
const cfg = {
id: DataTransformerID.ensureColumns,
options: {},
};
const data = [seriesBC];
const filtered = transformDataFrame([cfg], data);
expect(filtered).toEqual(data);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], data),
expect: filtered => {
expect(filtered).toEqual(data);
},
done,
});
});
});

28
packages/grafana-data/src/transformations/transformers/ensureColumns.ts
View File

@ -1,25 +1,33 @@
import { of } from 'rxjs';
import { seriesToColumnsTransformer } from './seriesToColumns';
import { DataFrame } from '../../types/dataFrame';
import { getTimeField } from '../../dataframe/processDataFrame';
import { DataTransformerInfo } from '../../types/transformations';
import { DataTransformerID } from './ids';
import { mergeMap } from 'rxjs/operators';
export const ensureColumnsTransformer: DataTransformerInfo = {
id: DataTransformerID.ensureColumns,
name: 'Ensure Columns Transformer',
description: 'Will check if current data frames is series or columns. If in series it will convert to columns.',
transformer: () => (data: DataFrame[]) => {
// Assume timeseries should first be joined by time
const timeFieldName = findConsistentTimeFieldName(data);
operator: (options = {}) => source =>
source.pipe(
mergeMap(data => {
// Assume timeseries should first be joined by time
const timeFieldName = findConsistentTimeFieldName(data);
if (data.length > 1 && timeFieldName) {
return seriesToColumnsTransformer.transformer({
byField: timeFieldName,
})(data);
}
if (data.length > 1 && timeFieldName) {
return of(data).pipe(
seriesToColumnsTransformer.operator({
byField: timeFieldName,
})
);
}
return data;
},
return of(data);
})
),
};
/**

15
packages/grafana-data/src/transformations/transformers/filter.test.ts
View File

@ -5,6 +5,7 @@ import { FieldMatcherID } from '../matchers/ids';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { filterFieldsTransformer } from './filter';
import { transformDataFrame } from '../transformDataFrame';
import { observableTester } from '../../utils/tests/observableTester';
export const simpleSeriesWithTypes = toDataFrame({
fields: [
@ -20,7 +21,7 @@ describe('Filter Transformer', () => {
mockTransformationsRegistry([filterFieldsTransformer]);
});
it('filters by include', () => {
it('filters by include', done => {
const cfg = {
id: DataTransformerID.filterFields,
options: {
@ -28,8 +29,14 @@ describe('Filter Transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [simpleSeriesWithTypes])[0];
expect(filtered.fields.length).toBe(1);
expect(filtered.fields[0].name).toBe('D');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [simpleSeriesWithTypes]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(1);
expect(filtered.fields[0].name).toBe('D');
},
done,
});
});
});

98
packages/grafana-data/src/transformations/transformers/filter.ts
View File

@ -1,3 +1,5 @@
import { map } from 'rxjs/operators';
import { noopTransformer } from './noop';
import { DataFrame, Field } from '../../types/dataFrame';
import { DataTransformerID } from './ids';
@ -19,46 +21,48 @@ export const filterFieldsTransformer: DataTransformerInfo<FilterOptions> = {
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: FilterOptions) => {
operator: (options: FilterOptions) => source => {
if (!options.include && !options.exclude) {
return noopTransformer.transformer({});
return source.pipe(noopTransformer.operator({}));
}
const include = options.include ? getFieldMatcher(options.include) : null;
const exclude = options.exclude ? getFieldMatcher(options.exclude) : null;
return source.pipe(
map(data => {
const include = options.include ? getFieldMatcher(options.include) : null;
const exclude = options.exclude ? getFieldMatcher(options.exclude) : null;
return (data: DataFrame[]) => {
const processed: DataFrame[] = [];
for (const series of data) {
// Find the matching field indexes
const fields: Field[] = [];
for (let i = 0; i < series.fields.length; i++) {
const field = series.fields[i];
const processed: DataFrame[] = [];
for (const series of data) {
// Find the matching field indexes
const fields: Field[] = [];
for (let i = 0; i < series.fields.length; i++) {
const field = series.fields[i];
if (exclude) {
if (exclude(field, series, data)) {
continue;
if (exclude) {
if (exclude(field, series, data)) {
continue;
}
if (!include) {
fields.push(field);
}
}
if (!include) {
if (include && include(field, series, data)) {
fields.push(field);
}
}
if (include && include(field, series, data)) {
fields.push(field);
}
}
if (!fields.length) {
continue;
if (!fields.length) {
continue;
}
const copy = {
...series, // all the other properties
fields, // but a different set of fields
};
processed.push(copy);
}
const copy = {
...series, // all the other properties
fields, // but a different set of fields
};
processed.push(copy);
}
return processed;
};
return processed;
})
);
},
};
@ -72,30 +76,32 @@ export const filterFramesTransformer: DataTransformerInfo<FilterOptions> = {
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: FilterOptions) => {
operator: options => source => {
if (!options.include && !options.exclude) {
return noopTransformer.transformer({});
return source.pipe(noopTransformer.operator({}));
}
const include = options.include ? getFrameMatchers(options.include) : null;
const exclude = options.exclude ? getFrameMatchers(options.exclude) : null;
return source.pipe(
map(data => {
const include = options.include ? getFrameMatchers(options.include) : null;
const exclude = options.exclude ? getFrameMatchers(options.exclude) : null;
return (data: DataFrame[]) => {
const processed: DataFrame[] = [];
for (const series of data) {
if (exclude) {
if (exclude(series)) {
continue;
const processed: DataFrame[] = [];
for (const series of data) {
if (exclude) {
if (exclude(series)) {
continue;
}
if (!include) {
processed.push(series);
}
}
if (!include) {
if (include && include(series)) {
processed.push(series);
}
}
if (include && include(series)) {
processed.push(series);
}
}
return processed;
};
return processed;
})
);
},
};

139
packages/grafana-data/src/transformations/transformers/filterByName.test.ts
View File

@ -5,6 +5,7 @@ import { mockTransformationsRegistry } from '../../utils/tests/mockTransformatio
import { filterFieldsByNameTransformer } from './filterByName';
import { filterFieldsTransformer } from './filter';
import { transformDataFrame } from '../transformDataFrame';
import { observableTester } from '../../utils/tests/observableTester';
export const seriesWithNamesToMatch = toDataFrame({
fields: [
@ -20,18 +21,24 @@ describe('filterByName transformer', () => {
mockTransformationsRegistry([filterFieldsByNameTransformer, filterFieldsTransformer]);
});
it('returns original series if no options provided', () => {
it('returns original series if no options provided', done => {
const cfg = {
id: DataTransformerID.filterFields,
options: {},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(4);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(4);
},
done,
});
});
describe('respects', () => {
it('inclusion by pattern', () => {
it('inclusion by pattern', done => {
const cfg = {
id: DataTransformerID.filterFieldsByName,
options: {
@ -41,12 +48,18 @@ describe('filterByName transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('startsWithA');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('startsWithA');
},
done,
});
});
it('exclusion by pattern', () => {
it('exclusion by pattern', done => {
const cfg = {
id: DataTransformerID.filterFieldsByName,
options: {
@ -56,12 +69,18 @@ describe('filterByName transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('B');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('B');
},
done,
});
});
it('inclusion and exclusion by pattern', () => {
it('inclusion and exclusion by pattern', done => {
const cfg = {
id: DataTransformerID.filterFieldsByName,
options: {
@ -70,12 +89,18 @@ describe('filterByName transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(1);
expect(filtered.fields[0].name).toBe('B');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(1);
expect(filtered.fields[0].name).toBe('B');
},
done,
});
});
it('inclusion by names', () => {
it('inclusion by names', done => {
const cfg = {
id: DataTransformerID.filterFieldsByName,
options: {
@ -85,12 +110,18 @@ describe('filterByName transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('startsWithA');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('startsWithA');
},
done,
});
});
it('exclusion by names', () => {
it('exclusion by names', done => {
const cfg = {
id: DataTransformerID.filterFieldsByName,
options: {
@ -100,12 +131,18 @@ describe('filterByName transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('B');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('B');
},
done,
});
});
it('inclusion and exclusion by names', () => {
it('inclusion and exclusion by names', done => {
const cfg = {
id: DataTransformerID.filterFieldsByName,
options: {
@ -114,12 +151,18 @@ describe('filterByName transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(1);
expect(filtered.fields[0].name).toBe('B');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(1);
expect(filtered.fields[0].name).toBe('B');
},
done,
});
});
it('inclusion by both', () => {
it('inclusion by both', done => {
const cfg = {
id: DataTransformerID.filterFieldsByName,
options: {
@ -130,12 +173,18 @@ describe('filterByName transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('startsWithA');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('startsWithA');
},
done,
});
});
it('exclusion by both', () => {
it('exclusion by both', done => {
const cfg = {
id: DataTransformerID.filterFieldsByName,
options: {
@ -146,12 +195,18 @@ describe('filterByName transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('B');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(2);
expect(filtered.fields[0].name).toBe('B');
},
done,
});
});
it('inclusion and exclusion by both', () => {
it('inclusion and exclusion by both', done => {
const cfg = {
id: DataTransformerID.filterFieldsByName,
options: {
@ -160,9 +215,15 @@ describe('filterByName transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithNamesToMatch])[0];
expect(filtered.fields.length).toBe(1);
expect(filtered.fields[0].name).toBe('B');
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithNamesToMatch]),
expect: data => {
const filtered = data[0];
expect(filtered.fields.length).toBe(1);
expect(filtered.fields[0].name).toBe('B');
},
done,
});
});
});
});

17
packages/grafana-data/src/transformations/transformers/filterByName.ts
View File

@ -1,7 +1,7 @@
import { DataTransformerID } from './ids';
import { DataTransformerInfo, MatcherConfig } from '../../types/transformations';
import { FieldMatcherID } from '../matchers/ids';
import { FilterOptions, filterFieldsTransformer } from './filter';
import { filterFieldsTransformer } from './filter';
import { RegexpOrNamesMatcherOptions } from '../matchers/nameMatcher';
export interface FilterFieldsByNameTransformerOptions {
@ -19,14 +19,13 @@ export const filterFieldsByNameTransformer: DataTransformerInfo<FilterFieldsByNa
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: FilterFieldsByNameTransformerOptions) => {
const filterOptions: FilterOptions = {
include: getMatcherConfig(options.include),
exclude: getMatcherConfig(options.exclude),
};
return filterFieldsTransformer.transformer(filterOptions);
},
operator: options => source =>
source.pipe(
filterFieldsTransformer.operator({
include: getMatcherConfig(options.include),
exclude: getMatcherConfig(options.exclude),
})
),
};
const getMatcherConfig = (options?: RegexpOrNamesMatcherOptions): MatcherConfig | undefined => {

24
packages/grafana-data/src/transformations/transformers/filterByRefId.test.ts
View File

@ -3,6 +3,7 @@ import { toDataFrame } from '../../dataframe/processDataFrame';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { filterFramesByRefIdTransformer } from './filterByRefId';
import { transformDataFrame } from '../transformDataFrame';
import { observableTester } from '../../utils/tests/observableTester';
export const allSeries = [
toDataFrame({
@ -23,18 +24,24 @@ describe('filterByRefId transformer', () => {
beforeAll(() => {
mockTransformationsRegistry([filterFramesByRefIdTransformer]);
});
it('returns all series if no options provided', () => {
it('returns all series if no options provided', done => {
const cfg = {
id: DataTransformerID.filterByRefId,
options: {},
};
const filtered = transformDataFrame([cfg], allSeries);
expect(filtered.length).toBe(3);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], allSeries),
expect: filtered => {
expect(filtered.length).toBe(3);
},
done,
});
});
describe('respects', () => {
it('inclusion', () => {
it('inclusion', done => {
const cfg = {
id: DataTransformerID.filterByRefId,
options: {
@ -42,8 +49,13 @@ describe('filterByRefId transformer', () => {
},
};
const filtered = transformDataFrame([cfg], allSeries);
expect(filtered.map(f => f.refId)).toEqual(['A', 'B']);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], allSeries),
expect: filtered => {
expect(filtered.map(f => f.refId)).toEqual(['A', 'B']);
},
done,
});
});
});
});

6
packages/grafana-data/src/transformations/transformers/filterByRefId.ts
View File

@ -1,5 +1,5 @@
import { DataTransformerID } from './ids';
import { FilterOptions, filterFramesTransformer } from './filter';
import { filterFramesTransformer, FilterOptions } from './filter';
import { DataTransformerInfo } from '../../types/transformations';
import { FrameMatcherID } from '../matchers/ids';
@ -18,7 +18,7 @@ export const filterFramesByRefIdTransformer: DataTransformerInfo<FilterFramesByR
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: FilterFramesByRefIdTransformerOptions) => {
operator: options => source => {
const filterOptions: FilterOptions = {};
if (options.include) {
filterOptions.include = {
@ -33,6 +33,6 @@ export const filterFramesByRefIdTransformer: DataTransformerInfo<FilterFramesByR
};
}
return filterFramesTransformer.transformer(filterOptions);
return source.pipe(filterFramesTransformer.operator(filterOptions));
},
};

208
packages/grafana-data/src/transformations/transformers/groupBy.test.ts
View File

@ -1,5 +1,5 @@
import { toDataFrame } from '../../dataframe/processDataFrame';
import { groupByTransformer, GroupByTransformerOptions, GroupByOperationID } from './groupBy';
import { GroupByOperationID, groupByTransformer, GroupByTransformerOptions } from './groupBy';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { transformDataFrame } from '../transformDataFrame';
import { Field, FieldType } from '../../types';
@ -7,13 +7,14 @@ import { DataTransformerID } from './ids';
import { ArrayVector } from '../../vector';
import { ReducerID } from '../fieldReducer';
import { DataTransformerConfig } from '@grafana/data';
import { observableTester } from '../../utils/tests/observableTester';
describe('GroupBy transformer', () => {
beforeAll(() => {
mockTransformationsRegistry([groupByTransformer]);
});
it('should not apply transformation if config is missing group by fields', () => {
it('should not apply transformation if config is missing group by fields', done => {
const testSeries = toDataFrame({
name: 'A',
fields: [
@ -35,11 +36,16 @@ describe('GroupBy transformer', () => {
},
};
const result = transformDataFrame([cfg], [testSeries]);
expect(result[0]).toBe(testSeries);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [testSeries]),
expect: result => {
expect(result[0]).toBe(testSeries);
},
done,
});
});
it('should group values by message', () => {
it('should group values by message', done => {
const testSeries = toDataFrame({
name: 'A',
fields: [
@ -61,21 +67,25 @@ describe('GroupBy transformer', () => {
},
};
const result = transformDataFrame([cfg], [testSeries]);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [testSeries]),
expect: result => {
const expected: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
},
];
const expected: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
expect(result[0].fields).toEqual(expected);
},
];
expect(result[0].fields).toEqual(expected);
done,
});
});
it('should group values by message and summarize values', () => {
it('should group values by message and summarize values', done => {
const testSeries = toDataFrame({
name: 'A',
fields: [
@ -101,27 +111,31 @@ describe('GroupBy transformer', () => {
},
};
const result = transformDataFrame([cfg], [testSeries]);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [testSeries]),
expect: result => {
const expected: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
},
{
name: 'values (sum)',
type: FieldType.number,
values: new ArrayVector([1, 4, 9]),
config: {},
},
];
const expected: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
expect(result[0].fields).toEqual(expected);
},
{
name: 'values (sum)',
type: FieldType.number,
values: new ArrayVector([1, 4, 9]),
config: {},
},
];
expect(result[0].fields).toEqual(expected);
done,
});
});
it('should group by and compute a few calculations for each group of values', () => {
it('should group by and compute a few calculations for each group of values', done => {
const testSeries = toDataFrame({
name: 'A',
fields: [
@ -151,39 +165,43 @@ describe('GroupBy transformer', () => {
},
};
const result = transformDataFrame([cfg], [testSeries]);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [testSeries]),
expect: result => {
const expected: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
},
{
name: 'time (count)',
type: FieldType.number,
values: new ArrayVector([1, 2, 3]),
config: {},
},
{
name: 'time (last)',
type: FieldType.time,
values: new ArrayVector([3000, 5000, 8000]),
config: {},
},
{
name: 'values (sum)',
type: FieldType.number,
values: new ArrayVector([1, 4, 9]),
config: {},
},
];
const expected: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
expect(result[0].fields).toEqual(expected);
},
{
name: 'time (count)',
type: FieldType.number,
values: new ArrayVector([1, 2, 3]),
config: {},
},
{
name: 'time (last)',
type: FieldType.time,
values: new ArrayVector([3000, 5000, 8000]),
config: {},
},
{
name: 'values (sum)',
type: FieldType.number,
values: new ArrayVector([1, 4, 9]),
config: {},
},
];
expect(result[0].fields).toEqual(expected);
done,
});
});
it('should group values in data frames induvidually', () => {
it('should group values in data frames induvidually', done => {
const testSeries = [
toDataFrame({
name: 'A',
@ -219,39 +237,43 @@ describe('GroupBy transformer', () => {
},
};
const result = transformDataFrame([cfg], testSeries);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], testSeries),
expect: result => {
const expectedA: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
},
{
name: 'values (sum)',
type: FieldType.number,
values: new ArrayVector([1, 4, 9]),
config: {},
},
];
const expectedA: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
},
{
name: 'values (sum)',
type: FieldType.number,
values: new ArrayVector([1, 4, 9]),
config: {},
},
];
const expectedB: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
},
{
name: 'values (sum)',
type: FieldType.number,
values: new ArrayVector([0, 7, 8]),
config: {},
},
];
const expectedB: Field[] = [
{
name: 'message',
type: FieldType.string,
values: new ArrayVector(['one', 'two', 'three']),
config: {},
expect(result[0].fields).toEqual(expectedA);
expect(result[1].fields).toEqual(expectedB);
},
{
name: 'values (sum)',
type: FieldType.number,
values: new ArrayVector([0, 7, 8]),
config: {},
},
];
expect(result[0].fields).toEqual(expectedA);
expect(result[1].fields).toEqual(expectedB);
done,
});
});
});

213
packages/grafana-data/src/transformations/transformers/groupBy.ts
View File

@ -1,5 +1,7 @@
import { map } from 'rxjs/operators';
import { DataTransformerID } from './ids';
import { DataFrame, FieldType, Field } from '../../types/dataFrame';
import { DataFrame, Field, FieldType } from '../../types/dataFrame';
import { DataTransformerInfo } from '../../types/transformations';
import { getFieldDisplayName } from '../../field/fieldState';
import { ArrayVector } from '../../vector/ArrayVector';
@ -33,127 +35,128 @@ export const groupByTransformer: DataTransformerInfo<GroupByTransformerOptions>
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: GroupByTransformerOptions) => {
const hasValidConfig = Object.keys(options.fields).find(
name => options.fields[name].operation === GroupByOperationID.groupBy
);
operator: options => source =>
source.pipe(
map(data => {
const hasValidConfig = Object.keys(options.fields).find(
name => options.fields[name].operation === GroupByOperationID.groupBy
);
return (data: DataFrame[]) => {
if (!hasValidConfig) {
return data;
}
const processed: DataFrame[] = [];
for (const frame of data) {
const groupByFields: Field[] = [];
for (const field of frame.fields) {
if (shouldGroupOnField(field, options)) {
groupByFields.push(field);
}
if (!hasValidConfig) {
return data;
}
if (groupByFields.length === 0) {
continue; // No group by field in this frame, ignore the frame
}
const processed: DataFrame[] = [];
// Group the values by fields and groups so we can get all values for a
// group for a given field.
const valuesByGroupKey: Record<string, Record<string, MutableField>> = {};
for (let rowIndex = 0; rowIndex < frame.length; rowIndex++) {
const groupKey = String(groupByFields.map(field => field.values.get(rowIndex)));
const valuesByField = valuesByGroupKey[groupKey] ?? {};
for (const frame of data) {
const groupByFields: Field[] = [];
if (!valuesByGroupKey[groupKey]) {
valuesByGroupKey[groupKey] = valuesByField;
for (const field of frame.fields) {
if (shouldGroupOnField(field, options)) {
groupByFields.push(field);
}
}
for (let field of frame.fields) {
const fieldName = getFieldDisplayName(field);
if (groupByFields.length === 0) {
continue; // No group by field in this frame, ignore the frame
}
if (!valuesByField[fieldName]) {
valuesByField[fieldName] = {
name: fieldName,
type: field.type,
config: { ...field.config },
values: new ArrayVector(),
};
// Group the values by fields and groups so we can get all values for a
// group for a given field.
const valuesByGroupKey: Record<string, Record<string, MutableField>> = {};
for (let rowIndex = 0; rowIndex < frame.length; rowIndex++) {
const groupKey = String(groupByFields.map(field => field.values.get(rowIndex)));
const valuesByField = valuesByGroupKey[groupKey] ?? {};
if (!valuesByGroupKey[groupKey]) {
valuesByGroupKey[groupKey] = valuesByField;
}
valuesByField[fieldName].values.add(field.values.get(rowIndex));
}
}
for (let field of frame.fields) {
const fieldName = getFieldDisplayName(field);
const fields: Field[] = [];
const groupKeys = Object.keys(valuesByGroupKey);
if (!valuesByField[fieldName]) {
valuesByField[fieldName] = {
name: fieldName,
type: field.type,
config: { ...field.config },
values: new ArrayVector(),
};
}
for (const field of groupByFields) {
const values = new ArrayVector();
const fieldName = getFieldDisplayName(field);
for (let key of groupKeys) {
const valuesByField = valuesByGroupKey[key];
values.add(valuesByField[fieldName].values.get(0));
valuesByField[fieldName].values.add(field.values.get(rowIndex));
}
}
fields.push({
name: field.name,
type: field.type,
config: {
...field.config,
},
values: values,
const fields: Field[] = [];
const groupKeys = Object.keys(valuesByGroupKey);
for (const field of groupByFields) {
const values = new ArrayVector();
const fieldName = getFieldDisplayName(field);
for (let key of groupKeys) {
const valuesByField = valuesByGroupKey[key];
values.add(valuesByField[fieldName].values.get(0));
}
fields.push({
name: field.name,
type: field.type,
config: {
...field.config,
},
values: values,
});
}
// Then for each calculations configured, compute and add a new field (column)
for (const field of frame.fields) {
if (!shouldCalculateField(field, options)) {
continue;
}
const fieldName = getFieldDisplayName(field);
const aggregations = options.fields[fieldName].aggregations;
const valuesByAggregation: Record<string, any[]> = {};
for (const groupKey of groupKeys) {
const fieldWithValuesForGroup = valuesByGroupKey[groupKey][fieldName];
const results = reduceField({
field: fieldWithValuesForGroup,
reducers: aggregations,
});
for (const aggregation of aggregations) {
if (!Array.isArray(valuesByAggregation[aggregation])) {
valuesByAggregation[aggregation] = [];
}
valuesByAggregation[aggregation].push(results[aggregation]);
}
}
for (const aggregation of aggregations) {
const aggregationField: Field = {
name: `${fieldName} (${aggregation})`,
values: new ArrayVector(valuesByAggregation[aggregation]),
type: FieldType.other,
config: {},
};
aggregationField.type = detectFieldType(aggregation, field, aggregationField);
fields.push(aggregationField);
}
}
processed.push({
fields,
length: groupKeys.length,
});
}
// Then for each calculations configured, compute and add a new field (column)
for (const field of frame.fields) {
if (!shouldCalculateField(field, options)) {
continue;
}
const fieldName = getFieldDisplayName(field);
const aggregations = options.fields[fieldName].aggregations;
const valuesByAggregation: Record<string, any[]> = {};
for (const groupKey of groupKeys) {
const fieldWithValuesForGroup = valuesByGroupKey[groupKey][fieldName];
const results = reduceField({
field: fieldWithValuesForGroup,
reducers: aggregations,
});
for (const aggregation of aggregations) {
if (!Array.isArray(valuesByAggregation[aggregation])) {
valuesByAggregation[aggregation] = [];
}
valuesByAggregation[aggregation].push(results[aggregation]);
}
}
for (const aggregation of aggregations) {
const aggregationField: Field = {
name: `${fieldName} (${aggregation})`,
values: new ArrayVector(valuesByAggregation[aggregation]),
type: FieldType.other,
config: {},
};
aggregationField.type = detectFieldType(aggregation, field, aggregationField);
fields.push(aggregationField);
}
}
processed.push({
fields,
length: groupKeys.length,
});
}
return processed;
};
},
return processed;
})
),
};
const shouldGroupOnField = (field: Field, options: GroupByTransformerOptions): boolean => {

93
packages/grafana-data/src/transformations/transformers/labelsToFields.test.ts
View File

@ -1,16 +1,17 @@
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { LabelsToFieldsOptions, labelsToFieldsTransformer } from './labelsToFields';
import { DataTransformerConfig, FieldType, FieldDTO } from '../../types';
import { DataTransformerConfig, FieldDTO, FieldType } from '../../types';
import { DataTransformerID } from './ids';
import { toDataFrame, toDataFrameDTO } from '../../dataframe';
import { transformDataFrame } from '../transformDataFrame';
import { observableTester } from '../../utils/tests/observableTester';
describe('Labels as Columns', () => {
beforeAll(() => {
mockTransformationsRegistry([labelsToFieldsTransformer]);
});
it('data frame with two labels', () => {
it('data frame with two labels', done => {
const cfg: DataTransformerConfig<LabelsToFieldsOptions> = {
id: DataTransformerID.labelsToFields,
options: {},
@ -24,23 +25,30 @@ describe('Labels as Columns', () => {
],
});
const result = toDataFrameDTO(transformDataFrame([cfg], [source])[0]);
const expected: FieldDTO[] = [
{ name: 'time', type: FieldType.time, values: [1000, 2000], config: {} },
{
name: 'location',
type: FieldType.string,
values: ['inside', 'inside'],
config: {},
},
{ name: 'feelsLike', type: FieldType.string, values: ['ok', 'ok'], config: {} },
{ name: 'Value', type: FieldType.number, values: [1, 2], config: {} },
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [source]),
expect: data => {
const result = toDataFrameDTO(data[0]);
expect(result.fields).toEqual(expected);
const expected: FieldDTO[] = [
{ name: 'time', type: FieldType.time, values: [1000, 2000], config: {} },
{
name: 'location',
type: FieldType.string,
values: ['inside', 'inside'],
config: {},
},
{ name: 'feelsLike', type: FieldType.string, values: ['ok', 'ok'], config: {} },
{ name: 'Value', type: FieldType.number, values: [1, 2], config: {} },
];
expect(result.fields).toEqual(expected);
},
done,
});
});
it('data frame with two labels and valueLabel option', () => {
it('data frame with two labels and valueLabel option', done => {
const cfg: DataTransformerConfig<LabelsToFieldsOptions> = {
id: DataTransformerID.labelsToFields,
options: { valueLabel: 'name' },
@ -63,22 +71,29 @@ describe('Labels as Columns', () => {
],
});
const result = toDataFrameDTO(transformDataFrame([cfg], [source])[0]);
const expected: FieldDTO[] = [
{ name: 'time', type: FieldType.time, values: [1000, 2000], config: {} },
{
name: 'location',
type: FieldType.string,
values: ['inside', 'inside'],
config: {},
},
{ name: 'Request', type: FieldType.number, values: [1, 2], config: {} },
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [source]),
expect: data => {
const result = toDataFrameDTO(data[0]);
expect(result.fields).toEqual(expected);
const expected: FieldDTO[] = [
{ name: 'time', type: FieldType.time, values: [1000, 2000], config: {} },
{
name: 'location',
type: FieldType.string,
values: ['inside', 'inside'],
config: {},
},
{ name: 'Request', type: FieldType.number, values: [1, 2], config: {} },
];
expect(result.fields).toEqual(expected);
},
done,
});
});
it('two data frames with 1 value and 1 label', () => {
it('two data frames with 1 value and 1 label', done => {
const cfg: DataTransformerConfig<LabelsToFieldsOptions> = {
id: DataTransformerID.labelsToFields,
options: {},
@ -100,14 +115,20 @@ describe('Labels as Columns', () => {
],
});
const result = toDataFrameDTO(transformDataFrame([cfg], [oneValueOneLabelA, oneValueOneLabelB])[0]);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [oneValueOneLabelA, oneValueOneLabelB]),
expect: data => {
const result = toDataFrameDTO(data[0]);
const expected: FieldDTO[] = [
{ name: 'time', type: FieldType.time, values: [1000, 2000], config: {} },
{ name: 'location', type: FieldType.string, values: ['inside', 'outside'], config: {} },
{ name: 'temp', type: FieldType.number, values: [1, -1], config: {} },
];
const expected: FieldDTO[] = [
{ name: 'time', type: FieldType.time, values: [1000, 2000], config: {} },
{ name: 'location', type: FieldType.string, values: ['inside', 'outside'], config: {} },
{ name: 'temp', type: FieldType.number, values: [1, -1], config: {} },
];
expect(result.fields).toEqual(expected);
expect(result.fields).toEqual(expected);
},
done,
});
});
});

94
packages/grafana-data/src/transformations/transformers/labelsToFields.ts
View File

@ -1,4 +1,6 @@
import { DataFrame, DataTransformerInfo, FieldType, Field } from '../../types';
import { map } from 'rxjs/operators';
import { DataFrame, DataTransformerInfo, Field, FieldType } from '../../types';
import { DataTransformerID } from './ids';
import { ArrayVector } from '../../vector';
import { mergeTransformer } from './merge';
@ -15,56 +17,60 @@ export const labelsToFieldsTransformer: DataTransformerInfo<LabelsToFieldsOption
name: 'Labels to fields',
description: 'Extract time series labels to fields (columns)',
defaultOptions: {},
transformer: options => (data: DataFrame[]) => {
const result: DataFrame[] = [];
operator: options => source =>
source.pipe(
map(data => {
const result: DataFrame[] = [];
for (const frame of data) {
const newFields: Field[] = [];
for (const frame of data) {
const newFields: Field[] = [];
for (const field of frame.fields) {
if (!field.labels) {
newFields.push(field);
continue;
}
for (const field of frame.fields) {
if (!field.labels) {
newFields.push(field);
continue;
}
let name = field.name;
let name = field.name;
for (const labelName of Object.keys(field.labels)) {
// if we should use this label as the value field name store it and skip adding this as a seperate field
if (options.valueLabel === labelName) {
name = field.labels[labelName];
continue;
for (const labelName of Object.keys(field.labels)) {
// if we should use this label as the value field name store it and skip adding this as a seperate field
if (options.valueLabel === labelName) {
name = field.labels[labelName];
continue;
}
const values = new Array(frame.length).fill(field.labels[labelName]);
newFields.push({
name: labelName,
type: FieldType.string,
values: new ArrayVector(values),
config: {},
});
}
// add the value field but clear out any labels or displayName
newFields.push({
...field,
name,
config: {
...field.config,
// we need to clear thes for this transform as these can contain label names that we no longer want
displayName: undefined,
displayNameFromDS: undefined,
},
labels: undefined,
});
}
const values = new Array(frame.length).fill(field.labels[labelName]);
newFields.push({
name: labelName,
type: FieldType.string,
values: new ArrayVector(values),
config: {},
result.push({
fields: newFields,
length: frame.length,
});
}
// add the value field but clear out any labels or displayName
newFields.push({
...field,
name,
config: {
...field.config,
// we need to clear thes for this transform as these can contain label names that we no longer want
displayName: undefined,
displayNameFromDS: undefined,
},
labels: undefined,
});
}
result.push({
fields: newFields,
length: frame.length,
});
}
return mergeTransformer.transformer({})(result);
},
return result;
}),
mergeTransformer.operator({})
),
};

393
packages/grafana-data/src/transformations/transformers/merge.test.ts
View File

@ -1,10 +1,11 @@
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { DataTransformerConfig, Field, FieldType, DisplayProcessor } from '../../types';
import { DataTransformerConfig, DisplayProcessor, Field, FieldType } from '../../types';
import { DataTransformerID } from './ids';
import { toDataFrame } from '../../dataframe';
import { transformDataFrame } from '../transformDataFrame';
import { ArrayVector } from '../../vector';
import { mergeTransformer, MergeTransformerOptions } from './merge';
import { observableTester } from '../../utils/tests/observableTester';
describe('Merge multipe to single', () => {
const cfg: DataTransformerConfig<MergeTransformerOptions> = {
@ -16,7 +17,7 @@ describe('Merge multipe to single', () => {
mockTransformationsRegistry([mergeTransformer]);
});
it('combine two series into one', () => {
it('combine two series into one', done => {
const seriesA = toDataFrame({
name: 'A',
fields: [
@ -33,16 +34,21 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [seriesA, seriesB]);
const expected: Field[] = [
createField('Time', FieldType.time, [1000, 2000]),
createField('Temp', FieldType.number, [1, -1]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesA, seriesB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [1000, 2000]),
createField('Temp', FieldType.number, [1, -1]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine two series with multiple values into one', () => {
it('combine two series with multiple values into one', done => {
const seriesA = toDataFrame({
name: 'A',
fields: [
@ -59,16 +65,21 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [seriesA, seriesB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesA, seriesB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine three series into one', () => {
it('combine three series into one', done => {
const seriesA = toDataFrame({
name: 'A',
fields: [
@ -93,16 +104,21 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [seriesA, seriesB, seriesC]);
const expected: Field[] = [
createField('Time', FieldType.time, [1000, 2000, 500]),
createField('Temp', FieldType.number, [1, -1, 2]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesA, seriesB, seriesC]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [1000, 2000, 500]),
createField('Temp', FieldType.number, [1, -1, 2]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine one serie and two tables into one table', () => {
it('combine one serie and two tables into one table', done => {
const tableA = toDataFrame({
name: 'A',
fields: [
@ -129,17 +145,22 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [tableA, seriesB, tableB]);
const expected: Field[] = [
createField('Time', FieldType.time, [1000, 1000, 500]),
createField('Temp', FieldType.number, [1, -1, 2]),
createField('Humidity', FieldType.number, [10, null, 5]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [tableA, seriesB, tableB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [1000, 1000, 500]),
createField('Temp', FieldType.number, [1, -1, 2]),
createField('Humidity', FieldType.number, [10, null, 5]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine one serie and two tables with ISO dates into one table', () => {
it('combine one serie and two tables with ISO dates into one table', done => {
const tableA = toDataFrame({
name: 'A',
fields: [
@ -166,17 +187,22 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [tableA, seriesB, tableC]);
const expected: Field[] = [
createField('Time', FieldType.time, ['2019-10-01T11:10:23Z', '2019-09-01T11:10:23Z', '2019-11-01T11:10:23Z']),
createField('Temp', FieldType.number, [1, -1, 2]),
createField('Humidity', FieldType.number, [10, null, 5]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [tableA, seriesB, tableC]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, ['2019-10-01T11:10:23Z', '2019-09-01T11:10:23Z', '2019-11-01T11:10:23Z']),
createField('Temp', FieldType.number, [1, -1, 2]),
createField('Humidity', FieldType.number, [10, null, 5]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine two tables, where first is partial overlapping, into one', () => {
it('combine two tables, where first is partial overlapping, into one', done => {
const tableA = toDataFrame({
name: 'A',
fields: [
@ -202,34 +228,39 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [tableA, tableB]);
const expected: Field[] = [
createField('Country', FieldType.string, [
'United States',
'United States',
'Mexico',
'Germany',
'Canada',
'Canada',
null,
]),
createField('AgeGroup', FieldType.string, [
'50 or over',
'35 - 49',
'0 - 17',
'35 - 49',
'35 - 49',
'25 - 34',
'18 - 24',
]),
createField('Sum', FieldType.number, [998, 1193, 1675, 146, 166, 219, null]),
createField('Count', FieldType.number, [2, 4, 1, 4, 4, 2, 3]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [tableA, tableB]),
expect: result => {
const expected: Field[] = [
createField('Country', FieldType.string, [
'United States',
'United States',
'Mexico',
'Germany',
'Canada',
'Canada',
null,
]),
createField('AgeGroup', FieldType.string, [
'50 or over',
'35 - 49',
'0 - 17',
'35 - 49',
'35 - 49',
'25 - 34',
'18 - 24',
]),
createField('Sum', FieldType.number, [998, 1193, 1675, 146, 166, 219, null]),
createField('Count', FieldType.number, [2, 4, 1, 4, 4, 2, 3]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine two tables, where second is partial overlapping, into one', () => {
it('combine two tables, where second is partial overlapping, into one', done => {
/**
* This behavior feels wrong. I would expect the same behavior regardless of the order
* of the frames. But when testing the old table panel it had this behavior so I am
@ -260,34 +291,39 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [tableA, tableB]);
const expected: Field[] = [
createField('AgeGroup', FieldType.string, [
'0 - 17',
'18 - 24',
'25 - 34',
'35 - 49',
'50 or over',
'35 - 49',
'35 - 49',
]),
createField('Count', FieldType.number, [1, 3, 2, 4, 2, null, null]),
createField('Country', FieldType.string, [
'Mexico',
null,
'Canada',
'United States',
'United States',
'Germany',
'Canada',
]),
createField('Sum', FieldType.number, [1675, null, 219, 1193, 998, 146, 166]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [tableA, tableB]),
expect: result => {
const expected: Field[] = [
createField('AgeGroup', FieldType.string, [
'0 - 17',
'18 - 24',
'25 - 34',
'35 - 49',
'50 or over',
'35 - 49',
'35 - 49',
]),
createField('Count', FieldType.number, [1, 3, 2, 4, 2, null, null]),
createField('Country', FieldType.string, [
'Mexico',
null,
'Canada',
'United States',
'United States',
'Germany',
'Canada',
]),
createField('Sum', FieldType.number, [1675, null, 219, 1193, 998, 146, 166]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine three tables with multiple values into one', () => {
it('combine three tables with multiple values into one', done => {
const tableA = toDataFrame({
name: 'A',
fields: [
@ -315,19 +351,23 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [tableA, tableB, tableC]);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [tableA, tableB, tableC]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126, 100, 124, 149]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3, 1, 4, 5]),
createField('Humidity', FieldType.number, [10, 14, 55, null, null, null, 22, 25, 30]),
createField('Enabled', FieldType.boolean, [null, null, null, true, false, true, null, null, null]),
];
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126, 100, 124, 149]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3, 1, 4, 5]),
createField('Humidity', FieldType.number, [10, 14, 55, null, null, null, 22, 25, 30]),
createField('Enabled', FieldType.boolean, [null, null, null, true, false, true, null, null, null]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine two time series, where first serie fields has displayName, into one', () => {
it('combine two time series, where first serie fields has displayName, into one', done => {
const serieA = toDataFrame({
name: 'A',
fields: [
@ -344,19 +384,24 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]),
];
const fields = unwrap(result[0].fields);
const fields = unwrap(result[0].fields);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
},
done,
});
});
it('combine two time series, where first serie fields has display processor, into one', () => {
it('combine two time series, where first serie fields has display processor, into one', done => {
const displayProcessor: DisplayProcessor = jest.fn();
const serieA = toDataFrame({
@ -375,19 +420,24 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126], {}, displayProcessor),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126], {}, displayProcessor),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]),
];
const fields = unwrap(result[0].fields);
const fields = unwrap(result[0].fields);
expect(fields[0].display).toBe(displayProcessor);
expect(fields).toEqual(expected);
expect(fields[0].display).toBe(displayProcessor);
expect(fields).toEqual(expected);
},
done,
});
});
it('combine two time series, where first serie fields has units, into one', () => {
it('combine two time series, where first serie fields has units, into one', done => {
const serieA = toDataFrame({
name: 'A',
fields: [
@ -404,19 +454,24 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3], { units: 'celsius' }),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3], { units: 'celsius' }),
];
const fields = unwrap(result[0].fields);
const fields = unwrap(result[0].fields);
expect(fields[1].config).toEqual({ units: 'celsius' });
expect(fields).toEqual(expected);
expect(fields[1].config).toEqual({ units: 'celsius' });
expect(fields).toEqual(expected);
},
done,
});
});
it('combine two time series, where second serie fields has units, into one', () => {
it('combine two time series, where second serie fields has units, into one', done => {
const serieA = toDataFrame({
name: 'A',
fields: [
@ -433,19 +488,24 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200, 100, 125, 126]),
createField('Temp', FieldType.number, [1, 4, 5, -1, 2, 3]),
];
const fields = unwrap(result[0].fields);
const fields = unwrap(result[0].fields);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
},
done,
});
});
it('combine one regular serie with an empty serie should return the regular serie', () => {
it('combine one regular serie with an empty serie should return the regular serie', done => {
const serieA = toDataFrame({
name: 'A',
fields: [
@ -459,19 +519,24 @@ describe('Merge multipe to single', () => {
fields: [],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200]),
createField('Temp', FieldType.number, [1, 4, 5]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200]),
createField('Temp', FieldType.number, [1, 4, 5]),
];
const fields = unwrap(result[0].fields);
const fields = unwrap(result[0].fields);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
},
done,
});
});
it('combine two regular series with an empty serie should return the combination of the regular series', () => {
it('combine two regular series with an empty serie should return the combination of the regular series', done => {
const serieA = toDataFrame({
name: 'A',
fields: [
@ -493,20 +558,25 @@ describe('Merge multipe to single', () => {
],
});
const result = transformDataFrame([cfg], [serieA, serieB, serieC]);
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200]),
createField('Temp', FieldType.number, [1, 4, 5]),
createField('Humidity', FieldType.number, [6, 7, 8]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB, serieC]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [100, 150, 200]),
createField('Temp', FieldType.number, [1, 4, 5]),
createField('Humidity', FieldType.number, [6, 7, 8]),
];
const fields = unwrap(result[0].fields);
const fields = unwrap(result[0].fields);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
expect(fields[1].config).toEqual({});
expect(fields).toEqual(expected);
},
done,
});
});
it('combine multiple empty series should return one empty serie', () => {
it('combine multiple empty series should return one empty serie', done => {
const serieA = toDataFrame({
name: 'A',
fields: [],
@ -522,12 +592,17 @@ describe('Merge multipe to single', () => {
fields: [],
});
const result = transformDataFrame([cfg], [serieA, serieB, serieC]);
const expected: Field[] = [];
const fields = unwrap(result[0].fields);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB, serieC]),
expect: result => {
const expected: Field[] = [];
const fields = unwrap(result[0].fields);
expect(fields).toEqual(expected);
expect(result.length).toEqual(1);
expect(fields).toEqual(expected);
expect(result.length).toEqual(1);
},
done,
});
});
});

157
packages/grafana-data/src/transformations/transformers/merge.ts
View File

@ -1,3 +1,5 @@
import { map } from 'rxjs/operators';
import { DataTransformerID } from './ids';
import { DataTransformerInfo } from '../../types/transformations';
import { DataFrame, Field } from '../../types/dataFrame';
@ -17,97 +19,98 @@ export const mergeTransformer: DataTransformerInfo<MergeTransformerOptions> = {
name: 'Merge series/tables',
description: 'Merges multiple series/tables into a single serie/table',
defaultOptions: {},
transformer: (options: MergeTransformerOptions) => {
return (dataFrames: DataFrame[]) => {
if (!Array.isArray(dataFrames) || dataFrames.length === 0) {
return dataFrames;
}
const data = dataFrames.filter(frame => frame.fields.length > 0);
if (data.length === 0) {
return [dataFrames[0]];
}
const fieldNames = new Set<string>();
const fieldIndexByName: Record<string, Record<number, number>> = {};
const fieldNamesForKey: string[] = [];
const dataFrame = new MutableDataFrame();
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const field = frame.fields[fieldIndex];
if (!fieldNames.has(field.name)) {
dataFrame.addField(copyFieldStructure(field));
fieldNames.add(field.name);
}
fieldIndexByName[field.name] = fieldIndexByName[field.name] || {};
fieldIndexByName[field.name][frameIndex] = fieldIndex;
if (data.length - 1 !== frameIndex) {
continue;
}
if (fieldExistsInAllFrames(fieldIndexByName, field, data)) {
fieldNamesForKey.push(field.name);
}
operator: options => source =>
source.pipe(
map(dataFrames => {
if (!Array.isArray(dataFrames) || dataFrames.length === 0) {
return dataFrames;
}
}
if (fieldNamesForKey.length === 0) {
return dataFrames;
}
const data = dataFrames.filter(frame => frame.fields.length > 0);
const valuesByKey: Record<string, Array<Record<string, any>>> = {};
const valuesInOrder: ValuePointer[] = [];
const keyFactory = createKeyFactory(data, fieldIndexByName, fieldNamesForKey);
const valueMapper = createValueMapper(data, fieldNames, fieldIndexByName);
if (data.length === 0) {
return [dataFrames[0]];
}
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
const fieldNames = new Set<string>();
const fieldIndexByName: Record<string, Record<number, number>> = {};
const fieldNamesForKey: string[] = [];
const dataFrame = new MutableDataFrame();
for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) {
const key = keyFactory(frameIndex, valueIndex);
const value = valueMapper(frameIndex, valueIndex);
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
if (!Array.isArray(valuesByKey[key])) {
valuesByKey[key] = [value];
valuesInOrder.push(createPointer(key, valuesByKey));
continue;
}
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const field = frame.fields[fieldIndex];
let valueWasMerged = false;
valuesByKey[key] = valuesByKey[key].map(existing => {
if (!isMergable(existing, value)) {
return existing;
if (!fieldNames.has(field.name)) {
dataFrame.addField(copyFieldStructure(field));
fieldNames.add(field.name);
}
valueWasMerged = true;
return { ...existing, ...value };
});
if (!valueWasMerged) {
valuesByKey[key].push(value);
valuesInOrder.push(createPointer(key, valuesByKey));
fieldIndexByName[field.name] = fieldIndexByName[field.name] || {};
fieldIndexByName[field.name][frameIndex] = fieldIndex;
if (data.length - 1 !== frameIndex) {
continue;
}
if (fieldExistsInAllFrames(fieldIndexByName, field, data)) {
fieldNamesForKey.push(field.name);
}
}
}
}
for (const pointer of valuesInOrder) {
const value = valuesByKey[pointer.key][pointer.index];
if (value) {
dataFrame.add(value, false);
if (fieldNamesForKey.length === 0) {
return dataFrames;
}
}
return [dataFrame];
};
},
const valuesByKey: Record<string, Array<Record<string, any>>> = {};
const valuesInOrder: ValuePointer[] = [];
const keyFactory = createKeyFactory(data, fieldIndexByName, fieldNamesForKey);
const valueMapper = createValueMapper(data, fieldNames, fieldIndexByName);
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) {
const key = keyFactory(frameIndex, valueIndex);
const value = valueMapper(frameIndex, valueIndex);
if (!Array.isArray(valuesByKey[key])) {
valuesByKey[key] = [value];
valuesInOrder.push(createPointer(key, valuesByKey));
continue;
}
let valueWasMerged = false;
valuesByKey[key] = valuesByKey[key].map(existing => {
if (!isMergable(existing, value)) {
return existing;
}
valueWasMerged = true;
return { ...existing, ...value };
});
if (!valueWasMerged) {
valuesByKey[key].push(value);
valuesInOrder.push(createPointer(key, valuesByKey));
}
}
}
for (const pointer of valuesInOrder) {
const value = valuesByKey[pointer.key][pointer.index];
if (value) {
dataFrame.add(value, false);
}
}
return [dataFrame];
})
),
};
const copyFieldStructure = (field: Field): Field => {

5
packages/grafana-data/src/transformations/transformers/noop.ts
View File

@ -1,5 +1,4 @@
import { DataTransformerID } from './ids';
import { DataFrame } from '../../types/dataFrame';
import { DataTransformerInfo } from '../../types/transformations';
export interface NoopTransformerOptions {
@ -17,7 +16,5 @@ export const noopTransformer: DataTransformerInfo<NoopTransformerOptions> = {
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: NoopTransformerOptions) => {
return (data: DataFrame[]) => data;
},
operator: (options: NoopTransformerOptions) => source => source,
};

196
packages/grafana-data/src/transformations/transformers/order.test.ts
View File

@ -8,6 +8,7 @@ import {
} from '@grafana/data';
import { orderFieldsTransformer, OrderFieldsTransformerOptions } from './order';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { observableTester } from '../../utils/tests/observableTester';
describe('Order Transformer', () => {
beforeAll(() => {
@ -23,7 +24,7 @@ describe('Order Transformer', () => {
],
});
it('should order according to config', () => {
it('should order according to config', done => {
const cfg: DataTransformerConfig<OrderFieldsTransformerOptions> = {
id: DataTransformerID.order,
options: {
@ -35,40 +36,45 @@ describe('Order Transformer', () => {
},
};
const ordered = transformDataFrame([cfg], [data])[0];
expect(ordered.fields).toEqual([
{
config: {},
name: 'temperature',
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
labels: undefined,
state: {
displayName: 'temperature',
},
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [data]),
expect: data => {
const ordered = data[0];
expect(ordered.fields).toEqual([
{
config: {},
name: 'temperature',
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
labels: undefined,
state: {
displayName: 'temperature',
},
},
{
config: {},
name: 'humidity',
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
labels: undefined,
state: {
displayName: 'humidity',
},
},
{
config: {},
name: 'time',
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
labels: undefined,
state: {
displayName: 'time',
},
},
]);
},
{
config: {},
name: 'humidity',
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
labels: undefined,
state: {
displayName: 'humidity',
},
},
{
config: {},
name: 'time',
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
labels: undefined,
state: {
displayName: 'time',
},
},
]);
done,
});
});
});
@ -82,7 +88,7 @@ describe('Order Transformer', () => {
],
});
it('should append fields missing in config at the end', () => {
it('should append fields missing in config at the end', done => {
const cfg: DataTransformerConfig<OrderFieldsTransformerOptions> = {
id: DataTransformerID.order,
options: {
@ -94,40 +100,45 @@ describe('Order Transformer', () => {
},
};
const ordered = transformDataFrame([cfg], [data])[0];
expect(ordered.fields).toEqual([
{
config: {},
name: 'humidity',
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
labels: undefined,
state: {
displayName: 'humidity',
},
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [data]),
expect: data => {
const ordered = data[0];
expect(ordered.fields).toEqual([
{
config: {},
name: 'humidity',
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
labels: undefined,
state: {
displayName: 'humidity',
},
},
{
config: {},
name: 'time',
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
labels: undefined,
state: {
displayName: 'time',
},
},
{
config: {},
name: 'pressure',
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
labels: undefined,
state: {
displayName: 'pressure',
},
},
]);
},
{
config: {},
name: 'time',
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
labels: undefined,
state: {
displayName: 'time',
},
},
{
config: {},
name: 'pressure',
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
labels: undefined,
state: {
displayName: 'pressure',
},
},
]);
done,
});
});
});
@ -141,7 +152,7 @@ describe('Order Transformer', () => {
],
});
it('should keep the same order as in the incoming data', () => {
it('should keep the same order as in the incoming data', done => {
const cfg: DataTransformerConfig<OrderFieldsTransformerOptions> = {
id: DataTransformerID.order,
options: {
@ -149,28 +160,33 @@ describe('Order Transformer', () => {
},
};
const ordered = transformDataFrame([cfg], [data])[0];
expect(ordered.fields).toEqual([
{
config: {},
name: 'time',
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [data]),
expect: data => {
const ordered = data[0];
expect(ordered.fields).toEqual([
{
config: {},
name: 'time',
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
},
{
config: {},
name: 'pressure',
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
{
config: {},
name: 'humidity',
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
},
{
config: {},
name: 'pressure',
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
{
config: {},
name: 'humidity',
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
done,
});
});
});
});

26
packages/grafana-data/src/transformations/transformers/order.ts
View File

@ -2,6 +2,7 @@ import { DataTransformerID } from './ids';
import { DataTransformerInfo } from '../../types/transformations';
import { DataFrame, Field } from '../../types';
import { getFieldDisplayName } from '../../field/fieldState';
import { map } from 'rxjs/operators';
export interface OrderFieldsTransformerOptions {
indexByName: Record<string, number>;
@ -19,20 +20,21 @@ export const orderFieldsTransformer: DataTransformerInfo<OrderFieldsTransformerO
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: OrderFieldsTransformerOptions) => {
const orderer = createFieldsOrderer(options.indexByName);
operator: options => source =>
source.pipe(
map(data => {
const orderer = createFieldsOrderer(options.indexByName);
return (data: DataFrame[]) => {
if (!Array.isArray(data) || data.length === 0) {
return data;
}
if (!Array.isArray(data) || data.length === 0) {
return data;
}
return data.map(frame => ({
...frame,
fields: orderer(frame.fields, data, frame),
}));
};
},
return data.map(frame => ({
...frame,
fields: orderer(frame.fields, data, frame),
}));
})
),
};
export const createOrderFieldsComparer = (indexByName: Record<string, number>) => (a: string, b: string) => {

115
packages/grafana-data/src/transformations/transformers/organize.test.ts
View File

@ -8,6 +8,7 @@ import {
} from '@grafana/data';
import { organizeFieldsTransformer, OrganizeFieldsTransformerOptions } from './organize';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { observableTester } from '../../utils/tests/observableTester';
describe('OrganizeFields Transformer', () => {
beforeAll(() => {
@ -24,7 +25,7 @@ describe('OrganizeFields Transformer', () => {
],
});
it('should order and filter according to config', () => {
it('should order and filter according to config', done => {
const cfg: DataTransformerConfig<OrganizeFieldsTransformerOptions> = {
id: DataTransformerID.organize,
options: {
@ -42,32 +43,37 @@ describe('OrganizeFields Transformer', () => {
},
};
const organized = transformDataFrame([cfg], [data])[0];
expect(organized.fields).toEqual([
{
config: {},
labels: undefined,
name: 'temperature',
state: {
displayName: 'temperature',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [data]),
expect: data => {
const organized = data[0];
expect(organized.fields).toEqual([
{
config: {},
labels: undefined,
name: 'temperature',
state: {
displayName: 'temperature',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
{
config: {
displayName: 'renamed_humidity',
},
labels: undefined,
name: 'humidity',
state: {
displayName: 'renamed_humidity',
},
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
},
{
config: {
displayName: 'renamed_humidity',
},
labels: undefined,
name: 'humidity',
state: {
displayName: 'renamed_humidity',
},
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
done,
});
});
});
@ -81,7 +87,7 @@ describe('OrganizeFields Transformer', () => {
],
});
it('should append fields missing in config at the end', () => {
it('should append fields missing in config at the end', done => {
const cfg: DataTransformerConfig<OrganizeFieldsTransformerOptions> = {
id: DataTransformerID.organize,
options: {
@ -99,32 +105,37 @@ describe('OrganizeFields Transformer', () => {
},
};
const organized = transformDataFrame([cfg], [data])[0];
expect(organized.fields).toEqual([
{
labels: undefined,
config: {
displayName: 'renamed_time',
},
name: 'time',
state: {
displayName: 'renamed_time',
},
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [data]),
expect: data => {
const organized = data[0];
expect(organized.fields).toEqual([
{
labels: undefined,
config: {
displayName: 'renamed_time',
},
name: 'time',
state: {
displayName: 'renamed_time',
},
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
},
{
config: {},
labels: undefined,
name: 'pressure',
state: {
displayName: 'pressure',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
]);
},
{
config: {},
labels: undefined,
name: 'pressure',
state: {
displayName: 'pressure',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
]);
done,
});
});
});
});

22
packages/grafana-data/src/transformations/transformers/organize.ts
View File

@ -1,9 +1,8 @@
import { DataTransformerID } from './ids';
import { DataTransformerInfo } from '../../types/transformations';
import { OrderFieldsTransformerOptions, orderFieldsTransformer } from './order';
import { orderFieldsTransformer, OrderFieldsTransformerOptions } from './order';
import { filterFieldsByNameTransformer } from './filterByName';
import { DataFrame } from '../..';
import { RenameFieldsTransformerOptions, renameFieldsTransformer } from './rename';
import { renameFieldsTransformer, RenameFieldsTransformerOptions } from './rename';
export interface OrganizeFieldsTransformerOptions
extends OrderFieldsTransformerOptions,
@ -25,15 +24,14 @@ export const organizeFieldsTransformer: DataTransformerInfo<OrganizeFieldsTransf
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: OrganizeFieldsTransformerOptions) => {
const rename = renameFieldsTransformer.transformer(options);
const order = orderFieldsTransformer.transformer(options);
const filter = filterFieldsByNameTransformer.transformer({
exclude: { names: mapToExcludeArray(options.excludeByName) },
});
return (data: DataFrame[]) => rename(order(filter(data)));
},
operator: options => source =>
source.pipe(
filterFieldsByNameTransformer.operator({
exclude: { names: mapToExcludeArray(options.excludeByName) },
}),
orderFieldsTransformer.operator(options),
renameFieldsTransformer.operator(options)
),
};
const mapToExcludeArray = (excludeByName: Record<string, boolean>): string[] => {

321
packages/grafana-data/src/transformations/transformers/reduce.test.ts
View File

@ -6,6 +6,7 @@ import { reduceTransformer } from './reduce';
import { transformDataFrame } from '../transformDataFrame';
import { Field, FieldType } from '../../types';
import { ArrayVector } from '../../vector';
import { observableTester } from '../../utils/tests/observableTester';
const seriesAWithSingleField = toDataFrame({
name: 'A',
@ -46,187 +47,211 @@ describe('Reducer Transformer', () => {
mockTransformationsRegistry([reduceTransformer]);
});
it('reduces multiple data frames with many fields', () => {
it('reduces multiple data frames with many fields', done => {
const cfg = {
id: DataTransformerID.reduce,
options: {
reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.last],
},
};
const processed = transformDataFrame([cfg], [seriesAWithMultipleFields, seriesBWithMultipleFields]);
const expected: Field[] = [
{
name: 'Field',
type: FieldType.string,
values: new ArrayVector(['A temperature', 'A humidity', 'B temperature', 'B humidity']),
config: {},
},
{
name: 'First',
type: FieldType.number,
values: new ArrayVector([3, 10000.3, 1, 11000.1]),
config: {},
},
{
name: 'Min',
type: FieldType.number,
values: new ArrayVector([3, 10000.3, 1, 11000.1]),
config: {},
},
{
name: 'Max',
type: FieldType.number,
values: new ArrayVector([6, 10000.6, 7, 11000.7]),
config: {},
},
{
name: 'Last',
type: FieldType.number,
values: new ArrayVector([6, 10000.6, 7, 11000.7]),
config: {},
},
];
expect(processed.length).toEqual(1);
expect(processed[0].length).toEqual(4);
expect(processed[0].fields).toEqual(expected);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesAWithMultipleFields, seriesBWithMultipleFields]),
expect: processed => {
const expected: Field[] = [
{
name: 'Field',
type: FieldType.string,
values: new ArrayVector(['A temperature', 'A humidity', 'B temperature', 'B humidity']),
config: {},
},
{
name: 'First',
type: FieldType.number,
values: new ArrayVector([3, 10000.3, 1, 11000.1]),
config: {},
},
{
name: 'Min',
type: FieldType.number,
values: new ArrayVector([3, 10000.3, 1, 11000.1]),
config: {},
},
{
name: 'Max',
type: FieldType.number,
values: new ArrayVector([6, 10000.6, 7, 11000.7]),
config: {},
},
{
name: 'Last',
type: FieldType.number,
values: new ArrayVector([6, 10000.6, 7, 11000.7]),
config: {},
},
];
expect(processed.length).toEqual(1);
expect(processed[0].length).toEqual(4);
expect(processed[0].fields).toEqual(expected);
},
done,
});
});
it('reduces multiple data frames with single field', () => {
it('reduces multiple data frames with single field', done => {
const cfg = {
id: DataTransformerID.reduce,
options: {
reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.last],
},
};
const processed = transformDataFrame([cfg], [seriesAWithSingleField, seriesBWithSingleField]);
const expected: Field[] = [
{
name: 'Field',
type: FieldType.string,
values: new ArrayVector(['A temperature', 'B temperature']),
config: {},
},
{
name: 'First',
type: FieldType.number,
values: new ArrayVector([3, 1]),
config: {},
},
{
name: 'Min',
type: FieldType.number,
values: new ArrayVector([3, 1]),
config: {},
},
{
name: 'Max',
type: FieldType.number,
values: new ArrayVector([6, 7]),
config: {},
},
{
name: 'Last',
type: FieldType.number,
values: new ArrayVector([6, 7]),
config: {},
},
];
expect(processed.length).toEqual(1);
expect(processed[0].length).toEqual(2);
expect(processed[0].fields).toEqual(expected);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesAWithSingleField, seriesBWithSingleField]),
expect: processed => {
const expected: Field[] = [
{
name: 'Field',
type: FieldType.string,
values: new ArrayVector(['A temperature', 'B temperature']),
config: {},
},
{
name: 'First',
type: FieldType.number,
values: new ArrayVector([3, 1]),
config: {},
},
{
name: 'Min',
type: FieldType.number,
values: new ArrayVector([3, 1]),
config: {},
},
{
name: 'Max',
type: FieldType.number,
values: new ArrayVector([6, 7]),
config: {},
},
{
name: 'Last',
type: FieldType.number,
values: new ArrayVector([6, 7]),
config: {},
},
];
expect(processed.length).toEqual(1);
expect(processed[0].length).toEqual(2);
expect(processed[0].fields).toEqual(expected);
},
done,
});
});
it('reduces single data frame with many fields', () => {
it('reduces single data frame with many fields', done => {
const cfg = {
id: DataTransformerID.reduce,
options: {
reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.last],
},
};
const processed = transformDataFrame([cfg], [seriesAWithMultipleFields]);
const expected: Field[] = [
{
name: 'Field',
type: FieldType.string,
values: new ArrayVector(['A temperature', 'A humidity']),
config: {},
},
{
name: 'First',
type: FieldType.number,
values: new ArrayVector([3, 10000.3]),
config: {},
},
{
name: 'Min',
type: FieldType.number,
values: new ArrayVector([3, 10000.3]),
config: {},
},
{
name: 'Max',
type: FieldType.number,
values: new ArrayVector([6, 10000.6]),
config: {},
},
{
name: 'Last',
type: FieldType.number,
values: new ArrayVector([6, 10000.6]),
config: {},
},
];
expect(processed.length).toEqual(1);
expect(processed[0].length).toEqual(2);
expect(processed[0].fields).toEqual(expected);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesAWithMultipleFields]),
expect: processed => {
const expected: Field[] = [
{
name: 'Field',
type: FieldType.string,
values: new ArrayVector(['A temperature', 'A humidity']),
config: {},
},
{
name: 'First',
type: FieldType.number,
values: new ArrayVector([3, 10000.3]),
config: {},
},
{
name: 'Min',
type: FieldType.number,
values: new ArrayVector([3, 10000.3]),
config: {},
},
{
name: 'Max',
type: FieldType.number,
values: new ArrayVector([6, 10000.6]),
config: {},
},
{
name: 'Last',
type: FieldType.number,
values: new ArrayVector([6, 10000.6]),
config: {},
},
];
expect(processed.length).toEqual(1);
expect(processed[0].length).toEqual(2);
expect(processed[0].fields).toEqual(expected);
},
done,
});
});
it('reduces single data frame with single field', () => {
it('reduces single data frame with single field', done => {
const cfg = {
id: DataTransformerID.reduce,
options: {
reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.last],
},
};
const processed = transformDataFrame([cfg], [seriesAWithSingleField]);
const expected: Field[] = [
{
name: 'Field',
type: FieldType.string,
values: new ArrayVector(['A temperature']),
config: {},
},
{
name: 'First',
type: FieldType.number,
values: new ArrayVector([3]),
config: {},
},
{
name: 'Min',
type: FieldType.number,
values: new ArrayVector([3]),
config: {},
},
{
name: 'Max',
type: FieldType.number,
values: new ArrayVector([6]),
config: {},
},
{
name: 'Last',
type: FieldType.number,
values: new ArrayVector([6]),
config: {},
},
];
expect(processed.length).toEqual(1);
expect(processed[0].length).toEqual(1);
expect(processed[0].fields).toEqual(expected);
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesAWithSingleField]),
expect: processed => {
const expected: Field[] = [
{
name: 'Field',
type: FieldType.string,
values: new ArrayVector(['A temperature']),
config: {},
},
{
name: 'First',
type: FieldType.number,
values: new ArrayVector([3]),
config: {},
},
{
name: 'Min',
type: FieldType.number,
values: new ArrayVector([3]),
config: {},
},
{
name: 'Max',
type: FieldType.number,
values: new ArrayVector([6]),
config: {},
},
{
name: 'Last',
type: FieldType.number,
values: new ArrayVector([6]),
config: {},
},
];
expect(processed.length).toEqual(1);
expect(processed[0].length).toEqual(1);
expect(processed[0].fields).toEqual(expected);
},
done,
});
});
});

146
packages/grafana-data/src/transformations/transformers/reduce.ts
View File

@ -1,3 +1,5 @@
import { map } from 'rxjs/operators';
import { DataTransformerID } from './ids';
import { DataTransformerInfo, MatcherConfig } from '../../types/transformations';
import { fieldReducers, reduceField, ReducerID } from '../fieldReducer';
@ -28,85 +30,87 @@ export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> =
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: ReduceTransformerOptions) => {
const matcher = options.fields ? getFieldMatcher(options.fields) : alwaysFieldMatcher;
const calculators = options.reducers && options.reducers.length ? fieldReducers.list(options.reducers) : [];
const reducers = calculators.map(c => c.id);
operator: options => source =>
source.pipe(
map(data => {
const matcher = options.fields ? getFieldMatcher(options.fields) : alwaysFieldMatcher;
const calculators = options.reducers && options.reducers.length ? fieldReducers.list(options.reducers) : [];
const reducers = calculators.map(c => c.id);
return (data: DataFrame[]) => {
const processed: DataFrame[] = [];
const processed: DataFrame[] = [];
for (let seriesIndex = 0; seriesIndex < data.length; seriesIndex++) {
const series = data[seriesIndex];
const values: ArrayVector[] = [];
const fields: Field[] = [];
const byId: KeyValue<ArrayVector> = {};
values.push(new ArrayVector()); // The name
fields.push({
name: 'Field',
type: FieldType.string,
values: values[0],
config: {},
});
for (const info of calculators) {
const vals = new ArrayVector();
byId[info.id] = vals;
values.push(vals);
for (let seriesIndex = 0; seriesIndex < data.length; seriesIndex++) {
const series = data[seriesIndex];
const values: ArrayVector[] = [];
const fields: Field[] = [];
const byId: KeyValue<ArrayVector> = {};
values.push(new ArrayVector()); // The name
fields.push({
name: info.name,
type: FieldType.other, // UNKNOWN until after we call the functions
values: values[values.length - 1],
name: 'Field',
type: FieldType.string,
values: values[0],
config: {},
});
for (const info of calculators) {
const vals = new ArrayVector();
byId[info.id] = vals;
values.push(vals);
fields.push({
name: info.name,
type: FieldType.other, // UNKNOWN until after we call the functions
values: values[values.length - 1],
config: {},
});
}
for (let i = 0; i < series.fields.length; i++) {
const field = series.fields[i];
if (field.type === FieldType.time) {
continue;
}
if (matcher(field, series, data)) {
const results = reduceField({
field,
reducers,
});
// Update the name list
const fieldName = getFieldDisplayName(field, series, data);
values[0].buffer.push(fieldName);
for (const info of calculators) {
const v = results[info.id];
byId[info.id].buffer.push(v);
}
}
}
for (const f of fields) {
const t = guessFieldTypeForField(f);
if (t) {
f.type = t;
}
}
processed.push({
...series, // Same properties, different fields
fields,
length: values[0].length,
});
}
for (let i = 0; i < series.fields.length; i++) {
const field = series.fields[i];
if (field.type === FieldType.time) {
continue;
}
if (matcher(field, series, data)) {
const results = reduceField({
field,
reducers,
});
// Update the name list
const fieldName = getFieldDisplayName(field, series, data);
values[0].buffer.push(fieldName);
for (const info of calculators) {
const v = results[info.id];
byId[info.id].buffer.push(v);
}
}
}
for (const f of fields) {
const t = guessFieldTypeForField(f);
if (t) {
f.type = t;
}
}
processed.push({
...series, // Same properties, different fields
fields,
length: values[0].length,
});
}
const withoutTime = filterFieldsTransformer.transformer({ exclude: { id: FieldMatcherID.time } })(processed);
return mergeResults(withoutTime);
};
},
return processed;
}),
filterFieldsTransformer.operator({ exclude: { id: FieldMatcherID.time } }),
map(mergeResults)
),
};
const mergeResults = (data: DataFrame[]) => {

218
packages/grafana-data/src/transformations/transformers/rename.test.ts
View File

@ -6,8 +6,9 @@ import {
toDataFrame,
transformDataFrame,
} from '@grafana/data';
import { RenameFieldsTransformerOptions, renameFieldsTransformer } from './rename';
import { renameFieldsTransformer, RenameFieldsTransformerOptions } from './rename';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { observableTester } from '../../utils/tests/observableTester';
describe('Rename Transformer', () => {
beforeAll(() => {
@ -24,7 +25,7 @@ describe('Rename Transformer', () => {
],
});
it('should rename according to config', () => {
it('should rename according to config', done => {
const cfg: DataTransformerConfig<RenameFieldsTransformerOptions> = {
id: DataTransformerID.rename,
options: {
@ -36,46 +37,51 @@ describe('Rename Transformer', () => {
},
};
const renamed = transformDataFrame([cfg], [data])[0];
expect(renamed.fields).toEqual([
{
config: {
displayName: 'Total time',
},
labels: undefined,
name: 'time',
state: {
displayName: 'Total time',
},
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [data]),
expect: data => {
const renamed = data[0];
expect(renamed.fields).toEqual([
{
config: {
displayName: 'Total time',
},
labels: undefined,
name: 'time',
state: {
displayName: 'Total time',
},
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
},
{
config: {
displayName: 'how cold is it?',
},
labels: undefined,
name: 'temperature',
state: {
displayName: 'how cold is it?',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
{
config: {
displayName: 'Moistiness',
},
name: 'humidity',
labels: undefined,
state: {
displayName: 'Moistiness',
},
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
},
{
config: {
displayName: 'how cold is it?',
},
labels: undefined,
name: 'temperature',
state: {
displayName: 'how cold is it?',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
{
config: {
displayName: 'Moistiness',
},
name: 'humidity',
labels: undefined,
state: {
displayName: 'Moistiness',
},
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
done,
});
});
});
@ -89,7 +95,7 @@ describe('Rename Transformer', () => {
],
});
it('should not rename fields missing in config', () => {
it('should not rename fields missing in config', done => {
const cfg: DataTransformerConfig<RenameFieldsTransformerOptions> = {
id: DataTransformerID.rename,
options: {
@ -101,44 +107,49 @@ describe('Rename Transformer', () => {
},
};
const renamed = transformDataFrame([cfg], [data])[0];
expect(renamed.fields).toEqual([
{
config: {
displayName: 'ttl',
},
name: 'time',
labels: undefined,
state: {
displayName: 'ttl',
},
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [data]),
expect: data => {
const renamed = data[0];
expect(renamed.fields).toEqual([
{
config: {
displayName: 'ttl',
},
name: 'time',
labels: undefined,
state: {
displayName: 'ttl',
},
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
},
{
config: {},
labels: undefined,
name: 'pressure',
state: {
displayName: 'pressure',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
{
config: {
displayName: 'hum',
},
labels: undefined,
name: 'humidity',
state: {
displayName: 'hum',
},
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
},
{
config: {},
labels: undefined,
name: 'pressure',
state: {
displayName: 'pressure',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
{
config: {
displayName: 'hum',
},
labels: undefined,
name: 'humidity',
state: {
displayName: 'hum',
},
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
done,
});
});
});
@ -152,7 +163,7 @@ describe('Rename Transformer', () => {
],
});
it('should keep the same names as in the incoming data', () => {
it('should keep the same names as in the incoming data', done => {
const cfg: DataTransformerConfig<RenameFieldsTransformerOptions> = {
id: DataTransformerID.rename,
options: {
@ -160,28 +171,33 @@ describe('Rename Transformer', () => {
},
};
const renamed = transformDataFrame([cfg], [data])[0];
expect(renamed.fields).toEqual([
{
config: {},
name: 'time',
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [data]),
expect: data => {
const renamed = data[0];
expect(renamed.fields).toEqual([
{
config: {},
name: 'time',
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000]),
},
{
config: {},
name: 'pressure',
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
{
config: {},
name: 'humidity',
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
},
{
config: {},
name: 'pressure',
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6]),
},
{
config: {},
name: 'humidity',
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6]),
},
]);
done,
});
});
});
});

26
packages/grafana-data/src/transformations/transformers/rename.ts
View File

@ -2,6 +2,7 @@ import { DataTransformerID } from './ids';
import { DataTransformerInfo } from '../../types/transformations';
import { DataFrame, Field } from '../../types/dataFrame';
import { getFieldDisplayName } from '../../field/fieldState';
import { map } from 'rxjs/operators';
export interface RenameFieldsTransformerOptions {
renameByName: Record<string, string>;
@ -19,20 +20,21 @@ export const renameFieldsTransformer: DataTransformerInfo<RenameFieldsTransforme
* Return a modified copy of the series. If the transform is not or should not
* be applied, just return the input series
*/
transformer: (options: RenameFieldsTransformerOptions) => {
const renamer = createRenamer(options.renameByName);
operator: options => source =>
source.pipe(
map(data => {
const renamer = createRenamer(options.renameByName);
return (data: DataFrame[]) => {
if (!Array.isArray(data) || data.length === 0) {
return data;
}
if (!Array.isArray(data) || data.length === 0) {
return data;
}
return data.map(frame => ({
...frame,
fields: renamer(frame),
}));
};
},
return data.map(frame => ({
...frame,
fields: renamer(frame),
}));
})
),
};
const createRenamer = (renameByName: Record<string, string>) => (frame: DataFrame): Field[] => {

531
packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts
View File

@ -9,6 +9,7 @@ import {
} from '@grafana/data';
import { SeriesToColumnsOptions, seriesToColumnsTransformer } from './seriesToColumns';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { observableTester } from '../../utils/tests/observableTester';
describe('SeriesToColumns Transformer', () => {
beforeAll(() => {
@ -33,7 +34,7 @@ describe('SeriesToColumns Transformer', () => {
],
});
it('joins by time field', () => {
it('joins by time field', done => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
@ -41,62 +42,68 @@ describe('SeriesToColumns Transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: {
displayName: 'time',
},
type: FieldType.time,
values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
config: {},
labels: undefined,
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries]),
expect: data => {
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: {
displayName: 'time',
},
type: FieldType.time,
values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
config: {},
labels: undefined,
},
{
name: 'temperature',
state: {
displayName: 'temperature even',
},
type: FieldType.number,
values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'humidity',
state: {
displayName: 'humidity even',
},
type: FieldType.number,
values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'temperature',
state: {
displayName: 'temperature odd',
},
type: FieldType.number,
values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
config: {},
labels: { name: 'odd' },
},
{
name: 'humidity',
state: {
displayName: 'humidity odd',
},
type: FieldType.number,
values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
config: {},
labels: { name: 'odd' },
},
]);
},
{
name: 'temperature',
state: {
displayName: 'temperature even',
},
type: FieldType.number,
values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'humidity',
state: {
displayName: 'humidity even',
},
type: FieldType.number,
values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'temperature',
state: {
displayName: 'temperature odd',
},
type: FieldType.number,
values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
config: {},
labels: { name: 'odd' },
},
{
name: 'humidity',
state: {
displayName: 'humidity odd',
},
type: FieldType.number,
values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
config: {},
labels: { name: 'odd' },
},
]);
done,
});
});
it('joins by temperature field', () => {
it('joins by temperature field', done => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
@ -104,62 +111,68 @@ describe('SeriesToColumns Transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0];
expect(filtered.fields).toEqual([
{
name: 'temperature',
state: {
displayName: 'temperature',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6, 11.1, 11.3, 11.5, 11.7]),
config: {},
labels: undefined,
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries]),
expect: data => {
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'temperature',
state: {
displayName: 'temperature',
},
type: FieldType.number,
values: new ArrayVector([10.3, 10.4, 10.5, 10.6, 11.1, 11.3, 11.5, 11.7]),
config: {},
labels: undefined,
},
{
name: 'time',
state: {
displayName: 'time even',
},
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000, null, null, null, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'humidity',
state: {
displayName: 'humidity even',
},
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6, null, null, null, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'time',
state: {
displayName: 'time odd',
},
type: FieldType.time,
values: new ArrayVector([null, null, null, null, 1000, 3000, 5000, 7000]),
config: {},
labels: { name: 'odd' },
},
{
name: 'humidity',
state: {
displayName: 'humidity odd',
},
type: FieldType.number,
values: new ArrayVector([null, null, null, null, 11000.1, 11000.3, 11000.5, 11000.7]),
config: {},
labels: { name: 'odd' },
},
]);
},
{
name: 'time',
state: {
displayName: 'time even',
},
type: FieldType.time,
values: new ArrayVector([3000, 4000, 5000, 6000, null, null, null, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'humidity',
state: {
displayName: 'humidity even',
},
type: FieldType.number,
values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6, null, null, null, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'time',
state: {
displayName: 'time odd',
},
type: FieldType.time,
values: new ArrayVector([null, null, null, null, 1000, 3000, 5000, 7000]),
config: {},
labels: { name: 'odd' },
},
{
name: 'humidity',
state: {
displayName: 'humidity odd',
},
type: FieldType.number,
values: new ArrayVector([null, null, null, null, 11000.1, 11000.3, 11000.5, 11000.7]),
config: {},
labels: { name: 'odd' },
},
]);
done,
});
});
it('joins by time field in reverse order', () => {
it('joins by time field in reverse order', done => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
@ -171,59 +184,65 @@ describe('SeriesToColumns Transformer', () => {
everySecondSeries.fields[1].values = new ArrayVector(everySecondSeries.fields[1].values.toArray().reverse());
everySecondSeries.fields[2].values = new ArrayVector(everySecondSeries.fields[2].values.toArray().reverse());
const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: {
displayName: 'time',
},
type: FieldType.time,
values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
config: {},
labels: undefined,
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries]),
expect: data => {
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: {
displayName: 'time',
},
type: FieldType.time,
values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
config: {},
labels: undefined,
},
{
name: 'temperature',
state: {
displayName: 'temperature even',
},
type: FieldType.number,
values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'humidity',
state: {
displayName: 'humidity even',
},
type: FieldType.number,
values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'temperature',
state: {
displayName: 'temperature odd',
},
type: FieldType.number,
values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
config: {},
labels: { name: 'odd' },
},
{
name: 'humidity',
state: {
displayName: 'humidity odd',
},
type: FieldType.number,
values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
config: {},
labels: { name: 'odd' },
},
]);
},
{
name: 'temperature',
state: {
displayName: 'temperature even',
},
type: FieldType.number,
values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'humidity',
state: {
displayName: 'humidity even',
},
type: FieldType.number,
values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
config: {},
labels: { name: 'even' },
},
{
name: 'temperature',
state: {
displayName: 'temperature odd',
},
type: FieldType.number,
values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
config: {},
labels: { name: 'odd' },
},
{
name: 'humidity',
state: {
displayName: 'humidity odd',
},
type: FieldType.number,
values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
config: {},
labels: { name: 'odd' },
},
]);
done,
});
});
describe('Field names', () => {
@ -243,7 +262,7 @@ describe('SeriesToColumns Transformer', () => {
],
});
it('when dataframe and field share the same name then use the field name', () => {
it('when dataframe and field share the same name then use the field name', done => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
@ -251,45 +270,51 @@ describe('SeriesToColumns Transformer', () => {
},
};
const filtered = transformDataFrame([cfg], [seriesWithSameFieldAndDataFrameName, seriesB])[0];
const expected: Field[] = [
{
name: 'time',
state: {
displayName: 'time',
},
type: FieldType.time,
values: new ArrayVector([1000, 2000, 3000, 4000]),
config: {},
labels: undefined,
},
{
name: 'temperature',
type: FieldType.number,
values: new ArrayVector([1, 3, 5, 7]),
config: {},
state: {
displayName: 'temperature temperature',
},
labels: { name: 'temperature' },
},
{
name: 'temperature',
state: {
displayName: 'temperature B',
},
type: FieldType.number,
values: new ArrayVector([2, 4, 6, 8]),
config: {},
labels: { name: 'B' },
},
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesWithSameFieldAndDataFrameName, seriesB]),
expect: data => {
const filtered = data[0];
const expected: Field[] = [
{
name: 'time',
state: {
displayName: 'time',
},
type: FieldType.time,
values: new ArrayVector([1000, 2000, 3000, 4000]),
config: {},
labels: undefined,
},
{
name: 'temperature',
type: FieldType.number,
values: new ArrayVector([1, 3, 5, 7]),
config: {},
state: {
displayName: 'temperature temperature',
},
labels: { name: 'temperature' },
},
{
name: 'temperature',
state: {
displayName: 'temperature B',
},
type: FieldType.number,
values: new ArrayVector([2, 4, 6, 8]),
config: {},
labels: { name: 'B' },
},
];
expect(filtered.fields).toEqual(expected);
expect(filtered.fields).toEqual(expected);
},
done,
});
});
});
it('joins if fields are missing', () => {
it('joins if fields are missing', done => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
@ -318,36 +343,41 @@ describe('SeriesToColumns Transformer', () => {
],
});
const filtered = transformDataFrame([cfg], [frame1, frame2, frame3])[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: { displayName: 'time' },
type: FieldType.time,
values: new ArrayVector([1, 2, 3]),
config: {},
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [frame1, frame2, frame3]),
expect: data => {
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: { displayName: 'time' },
type: FieldType.time,
values: new ArrayVector([1, 2, 3]),
config: {},
},
{
name: 'temperature',
state: { displayName: 'temperature A' },
type: FieldType.number,
values: new ArrayVector([10, 11, 12]),
config: {},
labels: { name: 'A' },
},
{
name: 'temperature',
state: { displayName: 'temperature C' },
type: FieldType.number,
values: new ArrayVector([20, 22, 24]),
config: {},
labels: { name: 'C' },
},
]);
},
{
name: 'temperature',
state: { displayName: 'temperature A' },
type: FieldType.number,
values: new ArrayVector([10, 11, 12]),
config: {},
labels: { name: 'A' },
},
{
name: 'temperature',
state: { displayName: 'temperature C' },
type: FieldType.number,
values: new ArrayVector([20, 22, 24]),
config: {},
labels: { name: 'C' },
},
]);
done,
});
});
it('handles duplicate field name', () => {
it('handles duplicate field name', done => {
const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
id: DataTransformerID.seriesToColumns,
options: {
@ -369,32 +399,37 @@ describe('SeriesToColumns Transformer', () => {
],
});
const filtered = transformDataFrame([cfg], [frame1, frame2])[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: { displayName: 'time' },
type: FieldType.time,
values: new ArrayVector([1]),
config: {},
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [frame1, frame2]),
expect: data => {
const filtered = data[0];
expect(filtered.fields).toEqual([
{
name: 'time',
state: { displayName: 'time' },
type: FieldType.time,
values: new ArrayVector([1]),
config: {},
},
{
name: 'temperature',
state: { displayName: 'temperature 1' },
type: FieldType.number,
values: new ArrayVector([10]),
config: {},
labels: {},
},
{
name: 'temperature',
state: { displayName: 'temperature 2' },
type: FieldType.number,
values: new ArrayVector([20]),
config: {},
labels: {},
},
]);
},
{
name: 'temperature',
state: { displayName: 'temperature 1' },
type: FieldType.number,
values: new ArrayVector([10]),
config: {},
labels: {},
},
{
name: 'temperature',
state: { displayName: 'temperature 2' },
type: FieldType.number,
values: new ArrayVector([20]),
config: {},
labels: {},
},
]);
done,
});
});
});

145
packages/grafana-data/src/transformations/transformers/seriesToColumns.ts
View File

@ -1,3 +1,5 @@
import { map } from 'rxjs/operators';
import { DataFrame, DataTransformerInfo, Field } from '../../types';
import { DataTransformerID } from './ids';
import { MutableDataFrame } from '../../dataframe';
@ -17,64 +19,66 @@ export const seriesToColumnsTransformer: DataTransformerInfo<SeriesToColumnsOpti
defaultOptions: {
byField: DEFAULT_KEY_FIELD,
},
transformer: options => (data: DataFrame[]) => {
const keyFieldMatch = options.byField || DEFAULT_KEY_FIELD;
const allFields: FieldsToProcess[] = [];
operator: options => source =>
source.pipe(
map(data => {
const keyFieldMatch = options.byField || DEFAULT_KEY_FIELD;
const allFields: FieldsToProcess[] = [];
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
const keyField = findKeyField(frame, keyFieldMatch);
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
const keyField = findKeyField(frame, keyFieldMatch);
if (!keyField) {
continue;
}
if (!keyField) {
continue;
}
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const sourceField = frame.fields[fieldIndex];
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const sourceField = frame.fields[fieldIndex];
if (sourceField === keyField) {
continue;
if (sourceField === keyField) {
continue;
}
let labels = sourceField.labels ?? {};
if (frame.name) {
labels = { ...labels, name: frame.name };
}
allFields.push({
keyField,
sourceField,
newField: {
...sourceField,
state: null,
values: new ArrayVector([]),
labels,
},
});
}
}
let labels = sourceField.labels ?? {};
if (frame.name) {
labels = { ...labels, name: frame.name };
// if no key fields or more than one value field
if (allFields.length <= 1) {
return data;
}
allFields.push({
keyField,
sourceField,
newField: {
...sourceField,
state: null,
values: new ArrayVector([]),
labels,
},
const resultFrame = new MutableDataFrame();
resultFrame.addField({
...allFields[0].keyField,
values: new ArrayVector([]),
});
}
}
// if no key fields or more than one value field
if (allFields.length <= 1) {
return data;
}
for (const item of allFields) {
item.newField = resultFrame.addField(item.newField);
}
const resultFrame = new MutableDataFrame();
const keyFieldTitle = getFieldDisplayName(resultFrame.fields[0], resultFrame);
const byKeyField: { [key: string]: { [key: string]: any } } = {};
resultFrame.addField({
...allFields[0].keyField,
values: new ArrayVector([]),
});
for (const item of allFields) {
item.newField = resultFrame.addField(item.newField);
}
const keyFieldTitle = getFieldDisplayName(resultFrame.fields[0], resultFrame);
const byKeyField: { [key: string]: { [key: string]: any } } = {};
/*
/*
this loop creates a dictionary object that groups the key fields values
{
"key field first value as string" : {
@ -90,36 +94,37 @@ export const seriesToColumnsTransformer: DataTransformerInfo<SeriesToColumnsOpti
}
*/
for (let fieldIndex = 0; fieldIndex < allFields.length; fieldIndex++) {
const { sourceField, keyField, newField } = allFields[fieldIndex];
const newFieldTitle = getFieldDisplayName(newField, resultFrame);
for (let fieldIndex = 0; fieldIndex < allFields.length; fieldIndex++) {
const { sourceField, keyField, newField } = allFields[fieldIndex];
const newFieldTitle = getFieldDisplayName(newField, resultFrame);
for (let valueIndex = 0; valueIndex < sourceField.values.length; valueIndex++) {
const value = sourceField.values.get(valueIndex);
const keyValue = keyField.values.get(valueIndex);
for (let valueIndex = 0; valueIndex < sourceField.values.length; valueIndex++) {
const value = sourceField.values.get(valueIndex);
const keyValue = keyField.values.get(valueIndex);
if (!byKeyField[keyValue]) {
byKeyField[keyValue] = { [newFieldTitle]: value, [keyFieldTitle]: keyValue };
} else {
byKeyField[keyValue][newFieldTitle] = value;
if (!byKeyField[keyValue]) {
byKeyField[keyValue] = { [newFieldTitle]: value, [keyFieldTitle]: keyValue };
} else {
byKeyField[keyValue][newFieldTitle] = value;
}
}
}
}
}
const keyValueStrings = Object.keys(byKeyField);
for (let rowIndex = 0; rowIndex < keyValueStrings.length; rowIndex++) {
const keyValueAsString = keyValueStrings[rowIndex];
const keyValueStrings = Object.keys(byKeyField);
for (let rowIndex = 0; rowIndex < keyValueStrings.length; rowIndex++) {
const keyValueAsString = keyValueStrings[rowIndex];
for (let fieldIndex = 0; fieldIndex < resultFrame.fields.length; fieldIndex++) {
const field = resultFrame.fields[fieldIndex];
const otherColumnName = getFieldDisplayName(field, resultFrame);
const value = byKeyField[keyValueAsString][otherColumnName] ?? null;
field.values.add(value);
}
}
for (let fieldIndex = 0; fieldIndex < resultFrame.fields.length; fieldIndex++) {
const field = resultFrame.fields[fieldIndex];
const otherColumnName = getFieldDisplayName(field, resultFrame);
const value = byKeyField[keyValueAsString][otherColumnName] ?? null;
field.values.add(value);
}
}
return [resultFrame];
},
return [resultFrame];
})
),
};
function findKeyField(frame: DataFrame, matchTitle: string): Field | null {

139
packages/grafana-data/src/transformations/transformers/seriesToRows.test.ts
View File

@ -5,13 +5,14 @@ import { toDataFrame } from '../../dataframe';
import { transformDataFrame } from '../transformDataFrame';
import { ArrayVector } from '../../vector';
import { seriesToRowsTransformer, SeriesToRowsTransformerOptions } from './seriesToRows';
import { observableTester } from '../../utils/tests/observableTester';
describe('Series to rows', () => {
beforeAll(() => {
mockTransformationsRegistry([seriesToRowsTransformer]);
});
it('combine two series into one', () => {
it('combine two series into one', done => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
@ -33,17 +34,22 @@ describe('Series to rows', () => {
],
});
const result = transformDataFrame([cfg], [seriesA, seriesB]);
const expected: Field[] = [
createField('Time', FieldType.time, [2000, 1000]),
createField('Metric', FieldType.string, ['B', 'A']),
createField('Value', FieldType.number, [-1, 1]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesA, seriesB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [2000, 1000]),
createField('Metric', FieldType.string, ['B', 'A']),
createField('Value', FieldType.number, [-1, 1]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine two series with multiple values into one', () => {
it('combine two series with multiple values into one', done => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
@ -65,17 +71,22 @@ describe('Series to rows', () => {
],
});
const result = transformDataFrame([cfg], [seriesA, seriesB]);
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesA, seriesB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine three series into one', () => {
it('combine three series into one', done => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
@ -105,17 +116,22 @@ describe('Series to rows', () => {
],
});
const result = transformDataFrame([cfg], [seriesA, seriesB, seriesC]);
const expected: Field[] = [
createField('Time', FieldType.time, [2000, 1000, 500]),
createField('Metric', FieldType.string, ['B', 'A', 'C']),
createField('Value', FieldType.number, [-1, 1, 2]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [seriesA, seriesB, seriesC]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [2000, 1000, 500]),
createField('Metric', FieldType.string, ['B', 'A', 'C']),
createField('Value', FieldType.number, [-1, 1, 2]),
];
expect(unwrap(result[0].fields)).toEqual(expected);
expect(unwrap(result[0].fields)).toEqual(expected);
},
done,
});
});
it('combine two time series, where first serie fields has displayName, into one', () => {
it('combine two time series, where first serie fields has displayName, into one', done => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
@ -137,20 +153,25 @@ describe('Series to rows', () => {
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
const fields = unwrap(result[0].fields);
const fields = unwrap(result[0].fields);
expect(fields[2].config).toEqual({});
expect(fields).toEqual(expected);
expect(fields[2].config).toEqual({});
expect(fields).toEqual(expected);
},
done,
});
});
it('combine two time series, where first serie fields has units, into one', () => {
it('combine two time series, where first serie fields has units, into one', done => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
@ -172,20 +193,25 @@ describe('Series to rows', () => {
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1], { units: 'celsius' }),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1], { units: 'celsius' }),
];
const fields = unwrap(result[0].fields);
const fields = unwrap(result[0].fields);
expect(fields[2].config).toEqual({ units: 'celsius' });
expect(fields).toEqual(expected);
expect(fields[2].config).toEqual({ units: 'celsius' });
expect(fields).toEqual(expected);
},
done,
});
});
it('combine two time series, where second serie fields has units, into one', () => {
it('combine two time series, where second serie fields has units, into one', done => {
const cfg: DataTransformerConfig<SeriesToRowsTransformerOptions> = {
id: DataTransformerID.seriesToRows,
options: {},
@ -207,17 +233,22 @@ describe('Series to rows', () => {
],
});
const result = transformDataFrame([cfg], [serieA, serieB]);
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
observableTester().subscribeAndExpectOnNext({
observable: transformDataFrame([cfg], [serieA, serieB]),
expect: result => {
const expected: Field[] = [
createField('Time', FieldType.time, [200, 150, 126, 125, 100, 100]),
createField('Metric', FieldType.string, ['A', 'A', 'B', 'B', 'A', 'B']),
createField('Value', FieldType.number, [5, 4, 3, 2, 1, -1]),
];
const fields = unwrap(result[0].fields);
const fields = unwrap(result[0].fields);
expect(fields[2].config).toEqual({});
expect(fields).toEqual(expected);
expect(fields[2].config).toEqual({});
expect(fields).toEqual(expected);
},
done,
});
});
});

100
packages/grafana-data/src/transformations/transformers/seriesToRows.ts
View File

@ -1,13 +1,14 @@
import { omit } from 'lodash';
import { map } from 'rxjs/operators';
import { DataTransformerID } from './ids';
import { DataTransformerInfo } from '../../types/transformations';
import {
DataFrame,
Field,
FieldType,
TIME_SERIES_METRIC_FIELD_NAME,
TIME_SERIES_TIME_FIELD_NAME,
TIME_SERIES_VALUE_FIELD_NAME,
TIME_SERIES_METRIC_FIELD_NAME,
} from '../../types/dataFrame';
import { isTimeSeries } from '../../dataframe/utils';
import { MutableDataFrame, sortDataFrame } from '../../dataframe';
@ -21,68 +22,69 @@ export const seriesToRowsTransformer: DataTransformerInfo<SeriesToRowsTransforme
name: 'Series to rows',
description: 'Combines multiple series into a single serie and appends a column with metric name per value.',
defaultOptions: {},
transformer: (options: SeriesToRowsTransformerOptions) => {
return (data: DataFrame[]) => {
if (!Array.isArray(data) || data.length <= 1) {
return data;
}
operator: options => source =>
source.pipe(
map(data => {
if (!Array.isArray(data) || data.length <= 1) {
return data;
}
if (!isTimeSeries(data)) {
return data;
}
if (!isTimeSeries(data)) {
return data;
}
const timeFieldByIndex: Record<number, number> = {};
const targetFields = new Set<string>();
const dataFrame = new MutableDataFrame();
const metricField: Field = {
name: TIME_SERIES_METRIC_FIELD_NAME,
values: new ArrayVector(),
config: {},
type: FieldType.string,
};
const timeFieldByIndex: Record<number, number> = {};
const targetFields = new Set<string>();
const dataFrame = new MutableDataFrame();
const metricField: Field = {
name: TIME_SERIES_METRIC_FIELD_NAME,
values: new ArrayVector(),
config: {},
type: FieldType.string,
};
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const field = frame.fields[fieldIndex];
for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
const field = frame.fields[fieldIndex];
if (field.type === FieldType.time) {
timeFieldByIndex[frameIndex] = fieldIndex;
if (field.type === FieldType.time) {
timeFieldByIndex[frameIndex] = fieldIndex;
if (!targetFields.has(TIME_SERIES_TIME_FIELD_NAME)) {
dataFrame.addField(copyFieldStructure(field, TIME_SERIES_TIME_FIELD_NAME));
dataFrame.addField(metricField);
targetFields.add(TIME_SERIES_TIME_FIELD_NAME);
if (!targetFields.has(TIME_SERIES_TIME_FIELD_NAME)) {
dataFrame.addField(copyFieldStructure(field, TIME_SERIES_TIME_FIELD_NAME));
dataFrame.addField(metricField);
targetFields.add(TIME_SERIES_TIME_FIELD_NAME);
}
continue;
}
continue;
}
if (!targetFields.has(TIME_SERIES_VALUE_FIELD_NAME)) {
dataFrame.addField(copyFieldStructure(field, TIME_SERIES_VALUE_FIELD_NAME));
targetFields.add(TIME_SERIES_VALUE_FIELD_NAME);
if (!targetFields.has(TIME_SERIES_VALUE_FIELD_NAME)) {
dataFrame.addField(copyFieldStructure(field, TIME_SERIES_VALUE_FIELD_NAME));
targetFields.add(TIME_SERIES_VALUE_FIELD_NAME);
}
}
}
}
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
const frame = data[frameIndex];
for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) {
const timeFieldIndex = timeFieldByIndex[frameIndex];
const valueFieldIndex = timeFieldIndex === 0 ? 1 : 0;
for (let valueIndex = 0; valueIndex < frame.length; valueIndex++) {
const timeFieldIndex = timeFieldByIndex[frameIndex];
const valueFieldIndex = timeFieldIndex === 0 ? 1 : 0;
dataFrame.add({
[TIME_SERIES_TIME_FIELD_NAME]: frame.fields[timeFieldIndex].values.get(valueIndex),
[TIME_SERIES_METRIC_FIELD_NAME]: getFrameDisplayName(frame),
[TIME_SERIES_VALUE_FIELD_NAME]: frame.fields[valueFieldIndex].values.get(valueIndex),
});
dataFrame.add({
[TIME_SERIES_TIME_FIELD_NAME]: frame.fields[timeFieldIndex].values.get(valueIndex),
[TIME_SERIES_METRIC_FIELD_NAME]: getFrameDisplayName(frame),
[TIME_SERIES_VALUE_FIELD_NAME]: frame.fields[valueFieldIndex].values.get(valueIndex),
});
}
}
}
return [sortDataFrame(dataFrame, 0, true)];
};
},
return [sortDataFrame(dataFrame, 0, true)];
})
),
};
const copyFieldStructure = (field: Field, name: string): Field => {

6
packages/grafana-data/src/types/annotations.ts
View File

@ -1,6 +1,8 @@
import { Observable } from 'rxjs';
import { ComponentType } from 'react';
import { DataQuery, QueryEditorProps } from './datasource';
import { DataFrame } from './dataFrame';
import { ComponentType } from 'react';
/**
* This JSON object is stored in the dashboard json model.
@ -79,7 +81,7 @@ export interface AnnotationSupport<TQuery extends DataQuery = DataQuery, TAnno =
/**
* When the standard frame > event processing is insufficient, this allows explicit control of the mappings
*/
processEvents?(anno: TAnno, data: DataFrame[]): AnnotationEvent[] | undefined;
processEvents?(anno: TAnno, data: DataFrame[]): Observable<AnnotationEvent[] | undefined>;
/**
* Specify a custom QueryEditor for the annotation page. If not specified, the standard one will be used

6
packages/grafana-data/src/types/transformations.ts
View File

@ -1,17 +1,17 @@
import { MonoTypeOperatorFunction } from 'rxjs';
import { DataFrame, Field } from './dataFrame';
import { RegistryItemWithOptions } from '../utils/Registry';
/**
* Function that transform data frames (AKA transformer)
*/
export type DataTransformer = (data: DataFrame[]) => DataFrame[];
export interface DataTransformerInfo<TOptions = any> extends RegistryItemWithOptions {
/**
* Function that configures transformation and returns a transformer
* @param options
*/
transformer: (options: TOptions) => DataTransformer;
operator: (options: TOptions) => MonoTypeOperatorFunction<DataFrame[]>;
}
export interface DataTransformerConfig<TOptions = any> {

1
packages/grafana-data/src/utils/index.ts
View File

@ -17,3 +17,4 @@ export { locationUtil } from './location';
export { urlUtil, UrlQueryMap, UrlQueryValue } from './url';
export { DataLinkBuiltInVars } from './dataLinks';
export { DocsId } from './docs';
export { observableTester } from './tests/observableTester';

5
public/test/helpers/observableTester.ts → packages/grafana-data/src/utils/tests/observableTester.ts
View File

@ -2,7 +2,10 @@ import { Observable } from 'rxjs';
interface ObservableTester<T> {
observable: Observable<T>;
done: jest.DoneCallback;
done: {
(...args: any[]): any;
fail(error?: string | { message: string }): any;
};
}
interface SubscribeAndExpectOnNext<T> extends ObservableTester<T> {

115
public/app/core/components/TransformersUI/CalculateFieldTransformerEditor.tsx
View File

@ -1,25 +1,27 @@
import React, { ChangeEvent } from 'react';
import { of, OperatorFunction } from 'rxjs';
import { map } from 'rxjs/operators';
import {
BinaryOperationID,
binaryOperators,
DataFrame,
DataTransformerID,
FieldType,
getFieldDisplayName,
KeyValue,
ReducerID,
SelectableValue,
standardTransformers,
TransformerRegistyItem,
TransformerUIProps,
BinaryOperationID,
SelectableValue,
binaryOperators,
getFieldDisplayName,
} from '@grafana/data';
import { Select, StatsPicker, LegacyForms, Input, FilterPill, HorizontalGroup } from '@grafana/ui';
import { FilterPill, HorizontalGroup, Input, LegacyForms, Select, StatsPicker } from '@grafana/ui';
import {
CalculateFieldTransformerOptions,
BinaryOptions,
CalculateFieldMode,
CalculateFieldTransformerOptions,
getNameFromOptions,
ReduceOptions,
BinaryOptions,
} from '@grafana/data/src/transformations/transformers/calculateField';
import defaults from 'lodash/defaults';
@ -64,44 +66,67 @@ export class CalculateFieldTransformerEditor extends React.PureComponent<
private initOptions() {
const { options } = this.props;
const configuredOptions = options?.reduce?.include || [];
const input = standardTransformers.ensureColumnsTransformer.transformer(null)(this.props.input);
const allNames: string[] = [];
const byName: KeyValue<boolean> = {};
for (const frame of input) {
for (const field of frame.fields) {
if (field.type !== FieldType.number) {
continue;
}
const displayName = getFieldDisplayName(field, frame, input);
if (!byName[displayName]) {
byName[displayName] = true;
allNames.push(displayName);
}
}
}
if (configuredOptions.length) {
const options: string[] = [];
const selected: string[] = [];
for (const v of allNames) {
if (configuredOptions.includes(v)) {
selected.push(v);
}
options.push(v);
}
this.setState({
names: options,
selected: selected,
const subscription = of(this.props.input)
.pipe(
standardTransformers.ensureColumnsTransformer.operator(null),
this.extractAllNames(),
this.extractNamesAndSelected(configuredOptions)
)
.subscribe(({ selected, names }) => {
this.setState({ names, selected }, () => subscription.unsubscribe());
});
} else {
this.setState({ names: allNames, selected: [] });
}
}
private extractAllNames(): OperatorFunction<DataFrame[], string[]> {
return source =>
source.pipe(
map(input => {
const allNames: string[] = [];
const byName: KeyValue<boolean> = {};
for (const frame of input) {
for (const field of frame.fields) {
if (field.type !== FieldType.number) {
continue;
}
const displayName = getFieldDisplayName(field, frame, input);
if (!byName[displayName]) {
byName[displayName] = true;
allNames.push(displayName);
}
}
}
return allNames;
})
);
}
private extractNamesAndSelected(
configuredOptions: string[]
): OperatorFunction<string[], { names: string[]; selected: string[] }> {
return source =>
source.pipe(
map(allNames => {
if (!configuredOptions.length) {
return { names: allNames, selected: [] };
}
const names: string[] = [];
const selected: string[] = [];
for (const v of allNames) {
if (configuredOptions.includes(v)) {
selected.push(v);
}
names.push(v);
}
return { names, selected };
})
);
}
onToggleReplaceFields = () => {

17
public/app/features/annotations/annotations_srv.ts
View File

@ -11,19 +11,19 @@ import { DashboardModel } from '../dashboard/state';
import {
AnnotationEvent,
AppEvents,
CoreApp,
DataQueryRequest,
DataSourceApi,
PanelEvents,
rangeUtil,
DataQueryRequest,
CoreApp,
ScopedVars,
} from '@grafana/data';
import { getBackendSrv, getDataSourceSrv } from '@grafana/runtime';
import { appEvents } from 'app/core/core';
import { getTimeSrv } from '../dashboard/services/TimeSrv';
import { Observable, of } from 'rxjs';
import { map } from 'rxjs/operators';
import { AnnotationQueryResponse, AnnotationQueryOptions } from './types';
import { map, mergeMap } from 'rxjs/operators';
import { AnnotationQueryOptions, AnnotationQueryResponse } from './types';
import { standardAnnotationSupport } from './standardAnnotationSupport';
import { runRequest } from '../dashboard/state/runRequest';
@ -264,9 +264,12 @@ export function executeAnnotationQuery(
};
return runRequest(datasource, queryRequest).pipe(
map(panelData => {
const events = panelData.series ? processor.processEvents!(annotation, panelData.series) : [];
return { panelData, events };
mergeMap(panelData => {
if (!panelData.series) {
return of({ panelData, events: [] });
}
return processor.processEvents!(annotation, panelData.series).pipe(map(events => ({ panelData, events })));
})
);
}

159
public/app/features/annotations/standardAnnotationSupport.test.ts
View File

@ -1,8 +1,8 @@
import { toDataFrame, FieldType } from '@grafana/data';
import { FieldType, observableTester, toDataFrame } from '@grafana/data';
import { getAnnotationsFromData } from './standardAnnotationSupport';
describe('DataFrame to annotations', () => {
test('simple conversion', () => {
test('simple conversion', done => {
const frame = toDataFrame({
fields: [
{ type: FieldType.time, values: [1, 2, 3, 4, 5] },
@ -11,53 +11,48 @@ describe('DataFrame to annotations', () => {
],
});
const events = getAnnotationsFromData([frame]);
expect(events).toMatchInlineSnapshot(`
Array [
Object {
"color": "red",
"tags": Array [
"aaa",
"bbb",
],
"text": "t1",
"time": 1,
"type": "default",
},
Object {
"color": "red",
"tags": Array [
"bbb",
"ccc",
],
"text": "t2",
"time": 2,
"type": "default",
},
Object {
"color": "red",
"tags": Array [
"zyz",
],
"text": "t3",
"time": 3,
"type": "default",
},
Object {
"color": "red",
"time": 4,
"type": "default",
},
Object {
"color": "red",
"time": 5,
"type": "default",
},
]
`);
observableTester().subscribeAndExpectOnNext({
observable: getAnnotationsFromData([frame]),
expect: events => {
expect(events).toEqual([
{
color: 'red',
tags: ['aaa', 'bbb'],
text: 't1',
time: 1,
type: 'default',
},
{
color: 'red',
tags: ['bbb', 'ccc'],
text: 't2',
time: 2,
type: 'default',
},
{
color: 'red',
tags: ['zyz'],
text: 't3',
time: 3,
type: 'default',
},
{
color: 'red',
time: 4,
type: 'default',
},
{
color: 'red',
time: 5,
type: 'default',
},
]);
},
done,
});
});
test('explicit mappins', () => {
test('explicit mappins', done => {
const frame = toDataFrame({
fields: [
{ name: 'time1', values: [111, 222, 333] },
@ -67,40 +62,42 @@ describe('DataFrame to annotations', () => {
],
});
const events = getAnnotationsFromData([frame], {
text: { value: 'bbbbb' },
time: { value: 'time2' },
timeEnd: { value: 'time1' },
title: { value: 'aaaaa' },
observableTester().subscribeAndExpectOnNext({
observable: getAnnotationsFromData([frame], {
text: { value: 'bbbbb' },
time: { value: 'time2' },
timeEnd: { value: 'time1' },
title: { value: 'aaaaa' },
}),
expect: events => {
expect(events).toEqual([
{
color: 'red',
text: 'b1',
time: 100,
timeEnd: 111,
title: 'a1',
type: 'default',
},
{
color: 'red',
text: 'b2',
time: 200,
timeEnd: 222,
title: 'a2',
type: 'default',
},
{
color: 'red',
text: 'b3',
time: 300,
timeEnd: 333,
title: 'a3',
type: 'default',
},
]);
},
done,
});
expect(events).toMatchInlineSnapshot(`
Array [
Object {
"color": "red",
"text": "b1",
"time": 100,
"timeEnd": 111,
"title": "a1",
"type": "default",
},
Object {
"color": "red",
"text": "b2",
"time": 200,
"timeEnd": 222,
"title": "a2",
"type": "default",
},
Object {
"color": "red",
"text": "b3",
"time": 300,
"timeEnd": 333,
"title": "a3",
"type": "default",
},
]
`);
});
});

201
public/app/features/annotations/standardAnnotationSupport.ts
View File

@ -1,15 +1,17 @@
import { Observable, of, OperatorFunction } from 'rxjs';
import { map, mergeMap } from 'rxjs/operators';
import {
DataFrame,
AnnotationEvent,
AnnotationEventFieldSource,
AnnotationEventMappings,
AnnotationQuery,
AnnotationSupport,
standardTransformers,
FieldType,
DataFrame,
Field,
KeyValue,
AnnotationEvent,
AnnotationEventMappings,
FieldType,
getFieldDisplayName,
AnnotationEventFieldSource,
KeyValue,
standardTransformers,
} from '@grafana/data';
import isString from 'lodash/isString';
@ -50,16 +52,25 @@ export const standardAnnotationSupport: AnnotationSupport = {
/**
* Flatten all panel data into a single frame
*/
export function singleFrameFromPanelData(data: DataFrame[]): DataFrame | undefined {
if (!data?.length) {
return undefined;
}
if (data.length === 1) {
return data[0];
}
export function singleFrameFromPanelData(): OperatorFunction<DataFrame[], DataFrame | undefined> {
return source =>
source.pipe(
mergeMap(data => {
if (!data?.length) {
return of(undefined);
}
return standardTransformers.mergeTransformer.transformer({})(data)[0];
if (data.length === 1) {
return of(data[0]);
}
return of(data).pipe(
standardTransformers.mergeTransformer.operator({}),
map(d => d[0])
);
})
);
}
interface AnnotationEventFieldSetter {
@ -100,96 +111,102 @@ export const annotationEventNames: AnnotationFieldInfo[] = [
// { key: 'email' },
];
export function getAnnotationsFromData(data: DataFrame[], options?: AnnotationEventMappings): AnnotationEvent[] {
const frame = singleFrameFromPanelData(data);
if (!frame?.length) {
return [];
}
let hasTime = false;
let hasText = false;
const byName: KeyValue<Field> = {};
for (const f of frame.fields) {
const name = getFieldDisplayName(f, frame);
byName[name.toLowerCase()] = f;
}
if (!options) {
options = {};
}
const fields: AnnotationEventFieldSetter[] = [];
for (const evts of annotationEventNames) {
const opt = options[evts.key] || {}; //AnnotationEventFieldMapping
if (opt.source === AnnotationEventFieldSource.Skip) {
continue;
}
const setter: AnnotationEventFieldSetter = { key: evts.key, split: evts.split };
if (opt.source === AnnotationEventFieldSource.Text) {
setter.text = opt.value;
} else {
const lower = (opt.value || evts.key).toLowerCase();
setter.field = byName[lower];
if (!setter.field && evts.field) {
setter.field = evts.field(frame);
export function getAnnotationsFromData(
data: DataFrame[],
options?: AnnotationEventMappings
): Observable<AnnotationEvent[]> {
return of(data).pipe(
singleFrameFromPanelData(),
map(frame => {
if (!frame?.length) {
return [];
}
}
if (setter.field || setter.text) {
fields.push(setter);
if (setter.key === 'time') {
hasTime = true;
} else if (setter.key === 'text') {
hasText = true;
let hasTime = false;
let hasText = false;
const byName: KeyValue<Field> = {};
for (const f of frame.fields) {
const name = getFieldDisplayName(f, frame);
byName[name.toLowerCase()] = f;
}
}
}
if (!hasTime || !hasText) {
return []; // throw an error?
}
if (!options) {
options = {};
}
// Add each value to the string
const events: AnnotationEvent[] = [];
const fields: AnnotationEventFieldSetter[] = [];
for (let i = 0; i < frame.length; i++) {
const anno: AnnotationEvent = {
type: 'default',
color: 'red',
};
for (const evts of annotationEventNames) {
const opt = options[evts.key] || {}; //AnnotationEventFieldMapping
for (const f of fields) {
let v: any = undefined;
if (opt.source === AnnotationEventFieldSource.Skip) {
continue;
}
if (f.text) {
v = f.text; // TODO support templates!
} else if (f.field) {
v = f.field.values.get(i);
if (v !== undefined && f.regex) {
const match = f.regex.exec(v);
if (match) {
v = match[1] ? match[1] : match[0];
const setter: AnnotationEventFieldSetter = { key: evts.key, split: evts.split };
if (opt.source === AnnotationEventFieldSource.Text) {
setter.text = opt.value;
} else {
const lower = (opt.value || evts.key).toLowerCase();
setter.field = byName[lower];
if (!setter.field && evts.field) {
setter.field = evts.field(frame);
}
}
if (setter.field || setter.text) {
fields.push(setter);
if (setter.key === 'time') {
hasTime = true;
} else if (setter.key === 'text') {
hasText = true;
}
}
}
if (v !== null && v !== undefined) {
if (f.split && typeof v === 'string') {
v = v.split(',');
}
(anno as any)[f.key] = v;
if (!hasTime || !hasText) {
return []; // throw an error?
}
}
events.push(anno);
}
// Add each value to the string
const events: AnnotationEvent[] = [];
return events;
for (let i = 0; i < frame.length; i++) {
const anno: AnnotationEvent = {
type: 'default',
color: 'red',
};
for (const f of fields) {
let v: any = undefined;
if (f.text) {
v = f.text; // TODO support templates!
} else if (f.field) {
v = f.field.values.get(i);
if (v !== undefined && f.regex) {
const match = f.regex.exec(v);
if (match) {
v = match[1] ? match[1] : match[0];
}
}
}
if (v !== null && v !== undefined) {
if (f.split && typeof v === 'string') {
v = v.split(',');
}
(anno as any)[f.key] = v;
}
}
events.push(anno);
}
return events;
})
);
}

78
public/app/features/dashboard/components/Inspector/InspectDataTab.tsx
View File

@ -1,4 +1,5 @@
import React, { PureComponent } from 'react';
import AutoSizer from 'react-virtualized-auto-sizer';
import {
applyFieldOverrides,
applyRawFieldOverrides,
@ -12,7 +13,6 @@ import {
} from '@grafana/data';
import { Button, Container, Field, HorizontalGroup, Icon, Select, Switch, Table, VerticalGroup } from '@grafana/ui';
import { selectors } from '@grafana/e2e-selectors';
import AutoSizer from 'react-virtualized-auto-sizer';
import { getPanelInspectorStyles } from './styles';
import { config } from 'app/core/config';
@ -38,6 +38,7 @@ interface State {
transformId: DataTransformerID;
dataFrameIndex: number;
transformationOptions: Array<SelectableValue<DataTransformerID>>;
transformedData: DataFrame[];
}
export class InspectDataTab extends PureComponent<Props, State> {
@ -49,9 +50,38 @@ export class InspectDataTab extends PureComponent<Props, State> {
dataFrameIndex: 0,
transformId: DataTransformerID.noop,
transformationOptions: buildTransformationOptions(),
transformedData: props.data ?? [],
};
}
componentDidUpdate(prevProps: Props, prevState: State) {
if (!this.props.data) {
this.setState({ transformedData: [] });
return;
}
if (this.props.options.withTransforms) {
this.setState({ transformedData: this.props.data });
return;
}
if (prevProps.data !== this.props.data || prevState.transformId !== this.state.transformId) {
const currentTransform = this.state.transformationOptions.find(item => item.value === this.state.transformId);
if (currentTransform && currentTransform.transformer.id !== DataTransformerID.noop) {
const selectedDataFrame = this.state.selectedDataFrame;
const dataFrameIndex = this.state.dataFrameIndex;
const subscription = transformDataFrame([currentTransform.transformer], this.props.data).subscribe(data => {
this.setState({ transformedData: data, selectedDataFrame, dataFrameIndex }, () => subscription.unsubscribe());
});
return;
}
this.setState({ transformedData: this.props.data });
return;
}
}
exportCsv = (dataFrame: DataFrame) => {
const { panel } = this.props;
const { transformId } = this.state;
@ -73,47 +103,11 @@ export class InspectDataTab extends PureComponent<Props, State> {
dataFrameIndex: typeof item.value === 'number' ? item.value : 0,
selectedDataFrame: item.value!,
});
this.props.onOptionsChange({
...this.props.options,
});
};
getTransformedData(): DataFrame[] {
const { transformId, transformationOptions } = this.state;
const { data } = this.props;
if (!data) {
return [];
}
const currentTransform = transformationOptions.find(item => item.value === transformId);
if (currentTransform && currentTransform.transformer.id !== DataTransformerID.noop) {
return transformDataFrame([currentTransform.transformer], data);
}
return data;
}
getProcessedData(): DataFrame[] {
const { options } = this.props;
let data = this.props.data;
if (!data) {
return [];
}
if (this.state.transformId !== DataTransformerID.noop) {
data = this.getTransformedData();
}
// In case the transform removes the currently selected data frame
if (!data[this.state.dataFrameIndex]) {
this.setState({
dataFrameIndex: 0,
selectedDataFrame: 0,
});
}
const data = this.state.transformedData;
if (!options.withFieldConfig) {
return applyRawFieldOverrides(data);
@ -265,9 +259,9 @@ export class InspectDataTab extends PureComponent<Props, State> {
return <div>No Data</div>;
}
if (!dataFrames[dataFrameIndex]) {
return <div>Could not find the Data Frame</div>;
}
// let's make sure we don't try to render a frame that doesn't exists
const index = !dataFrames[dataFrameIndex] ? 0 : dataFrameIndex;
const data = dataFrames[index];
return (
<div className={styles.dataTabContent} aria-label={selectors.components.PanelInspector.Data.content}>
@ -292,7 +286,7 @@ export class InspectDataTab extends PureComponent<Props, State> {
return (
<div style={{ width, height }}>
<Table width={width} height={height} data={dataFrames[dataFrameIndex]} />
<Table width={width} height={height} data={data} />
</div>
);
}}

77
public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx
View File

@ -1,29 +1,80 @@
import React, { useContext } from 'react';
import React, { useEffect, useMemo, useState } from 'react';
import { mergeMap } from 'rxjs/operators';
import { css } from 'emotion';
import { Icon, JSONFormatter, ThemeContext } from '@grafana/ui';
import { GrafanaTheme, DataFrame } from '@grafana/data';
import { Icon, JSONFormatter, useStyles } from '@grafana/ui';
import {
DataFrame,
DataTransformerConfig,
GrafanaTheme,
transformDataFrame,
TransformerRegistyItem,
} from '@grafana/data';
import { selectors } from '@grafana/e2e-selectors';
import { TransformationsEditorTransformation } from './types';
interface TransformationEditorProps {
name: string;
description?: string;
editor?: JSX.Element;
input: DataFrame[];
output?: DataFrame[];
debugMode?: boolean;
index: number;
data: DataFrame[];
uiConfig: TransformerRegistyItem<any>;
configs: TransformationsEditorTransformation[];
onChange: (index: number, config: DataTransformerConfig) => void;
}
export const TransformationEditor = ({ editor, input, output, debugMode, name }: TransformationEditorProps) => {
const theme = useContext(ThemeContext);
const styles = getStyles(theme);
export const TransformationEditor = ({
debugMode,
index,
data,
uiConfig,
configs,
onChange,
}: TransformationEditorProps) => {
const styles = useStyles(getStyles);
const [input, setInput] = useState<DataFrame[]>([]);
const [output, setOutput] = useState<DataFrame[]>([]);
const config = useMemo(() => configs[index], [configs, index]);
useEffect(() => {
const inputTransforms = configs.slice(0, index).map(t => t.transformation);
const outputTransforms = configs.slice(index, index + 1).map(t => t.transformation);
const inputSubscription = transformDataFrame(inputTransforms, data).subscribe(setInput);
const outputSubscription = transformDataFrame(inputTransforms, data)
.pipe(mergeMap(before => transformDataFrame(outputTransforms, before)))
.subscribe(setOutput);
return function unsubscribe() {
inputSubscription.unsubscribe();
outputSubscription.unsubscribe();
};
}, [index, data, configs]);
const editor = useMemo(
() =>
React.createElement(uiConfig.editor, {
options: { ...uiConfig.transformation.defaultOptions, ...config.transformation.options },
input,
onChange: (opts: any) => {
onChange(index, { id: config.transformation.id, options: opts });
},
}),
[
uiConfig.editor,
uiConfig.transformation.defaultOptions,
config.transformation.id,
config.transformation.options,
input,
onChange,
]
);
return (
<div className={styles.editor} aria-label={selectors.components.TransformTab.transformationEditor(name)}>
<div className={styles.editor} aria-label={selectors.components.TransformTab.transformationEditor(uiConfig.name)}>
{editor}
{debugMode && (
<div
className={styles.debugWrapper}
aria-label={selectors.components.TransformTab.transformationEditorDebugger(name)}
aria-label={selectors.components.TransformTab.transformationEditorDebugger(uiConfig.name)}
>
<div className={styles.debug}>
<div className={styles.debugTitle}>Transformation input data</div>

34
public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx
View File

@ -1,27 +1,30 @@
import { DataFrame } from '@grafana/data';
import React, { useState } from 'react';
import { DataFrame, DataTransformerConfig, TransformerRegistyItem } from '@grafana/data';
import { HorizontalGroup } from '@grafana/ui';
import { TransformationEditor } from './TransformationEditor';
import { QueryOperationRow } from 'app/core/components/QueryOperationRow/QueryOperationRow';
import { QueryOperationAction } from 'app/core/components/QueryOperationRow/QueryOperationAction';
import { TransformationsEditorTransformation } from './types';
interface TransformationOperationRowProps {
id: string;
index: number;
name: string;
description?: string;
input: DataFrame[];
output: DataFrame[];
editor?: JSX.Element;
onRemove: () => void;
data: DataFrame[];
uiConfig: TransformerRegistyItem<any>;
configs: TransformationsEditorTransformation[];
onRemove: (index: number) => void;
onChange: (index: number, config: DataTransformerConfig) => void;
}
export const TransformationOperationRow: React.FC<TransformationOperationRowProps> = ({
children,
onRemove,
index,
id,
...props
data,
configs,
uiConfig,
onChange,
}) => {
const [showDebug, setShowDebug] = useState(false);
@ -37,14 +40,21 @@ export const TransformationOperationRow: React.FC<TransformationOperationRowProp
}}
/>
<QueryOperationAction title="Remove" icon="trash-alt" onClick={onRemove} />
<QueryOperationAction title="Remove" icon="trash-alt" onClick={() => onRemove(index)} />
</HorizontalGroup>
);
};
return (
<QueryOperationRow id={id} index={index} title={props.name} draggable actions={renderActions}>
<TransformationEditor {...props} debugMode={showDebug} />
<QueryOperationRow id={id} index={index} title={uiConfig.name} draggable actions={renderActions}>
<TransformationEditor
debugMode={showDebug}
index={index}
data={data}
configs={configs}
uiConfig={uiConfig}
onChange={onChange}
/>
</QueryOperationRow>
);
};

43
public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRows.tsx Normal file
View File

@ -0,0 +1,43 @@
import React from 'react';
import { DataFrame, DataTransformerConfig, standardTransformersRegistry } from '@grafana/data';
import { TransformationOperationRow } from './TransformationOperationRow';
import { TransformationsEditorTransformation } from './types';
interface TransformationOperationRowsProps {
data: DataFrame[];
configs: TransformationsEditorTransformation[];
onRemove: (index: number) => void;
onChange: (index: number, config: DataTransformerConfig) => void;
}
export const TransformationOperationRows: React.FC<TransformationOperationRowsProps> = ({
data,
onChange,
onRemove,
configs,
}) => {
return (
<>
{configs.map((t, i) => {
const uiConfig = standardTransformersRegistry.getIfExists(t.transformation.id);
if (!uiConfig) {
return null;
}
return (
<TransformationOperationRow
index={i}
id={`${t.id}`}
key={`${t.id}`}
data={data}
configs={configs}
uiConfig={uiConfig}
onRemove={onRemove}
onChange={onChange}
/>
);
})}
</>
);
};

62
public/app/features/dashboard/components/TransformationsEditor/TransformationsEditor.tsx
View File

@ -18,9 +18,7 @@ import {
PanelData,
SelectableValue,
standardTransformersRegistry,
transformDataFrame,
} from '@grafana/data';
import { TransformationOperationRow } from './TransformationOperationRow';
import { Card, CardProps } from '../../../../core/components/Card/Card';
import { css } from 'emotion';
import { selectors } from '@grafana/e2e-selectors';
@ -28,6 +26,8 @@ import { Unsubscribable } from 'rxjs';
import { PanelModel } from '../../state';
import { getDocsLink } from 'app/core/utils/docsLinks';
import { DragDropContext, Droppable, DropResult } from 'react-beautiful-dnd';
import { TransformationOperationRows } from './TransformationOperationRows';
import { TransformationsEditorTransformation } from './types';
import { PanelNotSupported } from '../PanelEditor/PanelNotSupported';
import { AppNotificationSeverity } from '../../../../types';
@ -35,11 +35,6 @@ interface TransformationsEditorProps {
panel: PanelModel;
}
interface TransformationsEditorTransformation {
transformation: DataTransformerConfig;
id: string;
}
interface State {
data: DataFrame[];
transformations: TransformationsEditorTransformation[];
@ -197,53 +192,12 @@ export class TransformationsEditor extends React.PureComponent<TransformationsEd
{provided => {
return (
<div ref={provided.innerRef} {...provided.droppableProps}>
{transformations.map((t, i) => {
// Transformations are not identified uniquely by any property apart from array index.
// For drag and drop to work we need to generate unique ids. This record stores counters for each transformation type
// based on which ids are generated
let editor;
const transformationUI = standardTransformersRegistry.getIfExists(t.transformation.id);
if (!transformationUI) {
return null;
}
const input = transformDataFrame(
transformations.slice(0, i).map(t => t.transformation),
data
);
const output = transformDataFrame(
transformations.slice(i).map(t => t.transformation),
input
);
if (transformationUI) {
editor = React.createElement(transformationUI.editor, {
options: { ...transformationUI.transformation.defaultOptions, ...t.transformation.options },
input,
onChange: (options: any) => {
this.onTransformationChange(i, {
id: t.transformation.id,
options,
});
},
});
}
return (
<TransformationOperationRow
index={i}
id={`${t.id}`}
key={`${t.id}`}
input={input || []}
output={output || []}
onRemove={() => this.onTransformationRemove(i)}
editor={editor}
name={transformationUI.name}
description={transformationUI.description}
/>
);
})}
<TransformationOperationRows
configs={transformations}
data={data}
onRemove={this.onTransformationRemove}
onChange={this.onTransformationChange}
/>
{provided.placeholder}
</div>
);

6
public/app/features/dashboard/components/TransformationsEditor/types.ts Normal file
View File

@ -0,0 +1,6 @@
import { DataTransformerConfig } from '@grafana/data';
export interface TransformationsEditorTransformation {
transformation: DataTransformerConfig;
id: string;
}

56
public/app/features/dashboard/state/PanelQueryRunner.ts
View File

@ -1,31 +1,31 @@
// Libraries
import { cloneDeep } from 'lodash';
import { ReplaySubject, Unsubscribable, Observable } from 'rxjs';
import { map } from 'rxjs/operators';
import { MonoTypeOperatorFunction, Observable, of, ReplaySubject, Unsubscribable } from 'rxjs';
import { map, mergeMap } from 'rxjs/operators';
// Services & Utils
import { getTemplateSrv } from '@grafana/runtime';
import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
import { runRequest, preProcessPanelData } from './runRequest';
import { runSharedRequest, isSharedDashboardQuery } from '../../../plugins/datasource/dashboard';
import { preProcessPanelData, runRequest } from './runRequest';
import { isSharedDashboardQuery, runSharedRequest } from '../../../plugins/datasource/dashboard';
// Types
import {
PanelData,
DataQuery,
applyFieldOverrides,
CoreApp,
DataConfigSource,
DataQuery,
DataQueryRequest,
DataSourceApi,
DataSourceJsonData,
TimeRange,
DataTransformerConfig,
transformDataFrame,
ScopedVars,
applyFieldOverrides,
DataConfigSource,
TimeZone,
LoadingState,
PanelData,
rangeUtil,
ScopedVars,
TimeRange,
TimeZone,
transformDataFrame,
} from '@grafana/data';
export interface QueryRunnerOptions<
@ -75,21 +75,10 @@ export class PanelQueryRunner {
const { withFieldConfig, withTransforms } = options;
return this.subject.pipe(
this.getTransformationsStream(withTransforms),
map((data: PanelData) => {
let processedData = data;
// Apply transformation
if (withTransforms) {
const transformations = this.dataConfigSource.getTransformations();
if (transformations && transformations.length > 0) {
processedData = {
...processedData,
series: transformDataFrame(transformations, data.series),
};
}
}
if (withFieldConfig) {
// Apply field defaults & overrides
const fieldConfig = this.dataConfigSource.getFieldOverrideOptions();
@ -113,6 +102,25 @@ export class PanelQueryRunner {
);
}
private getTransformationsStream = (withTransforms: boolean): MonoTypeOperatorFunction<PanelData> => {
return inputStream =>
inputStream.pipe(
mergeMap(data => {
if (!withTransforms) {
return of(data);
}
const transformations = this.dataConfigSource.getTransformations();
if (!transformations || transformations.length === 0) {
return of(data);
}
return transformDataFrame(transformations, data.series).pipe(map(series => ({ ...data, series })));
})
);
};
async run(options: QueryRunnerOptions) {
const {
queries,

7
public/app/features/explore/state/actionTypes.ts
View File

@ -10,12 +10,11 @@ import {
HistoryItem,
LoadingState,
LogLevel,
PanelData,
LogsDedupStrategy,
QueryFixAction,
TimeRange,
LogsDedupStrategy,
} from '@grafana/data';
import { ExploreId, ExploreItemState } from 'app/types/explore';
import { ExploreId, ExploreItemState, ExplorePanelData } from 'app/types/explore';
export interface AddQueryRowPayload {
exploreId: ExploreId;
@ -82,7 +81,7 @@ export interface ModifyQueriesPayload {
export interface QueryEndedPayload {
exploreId: ExploreId;
response: PanelData;
response: ExplorePanelData;
}
export interface QueryStoreSubscriptionPayload {

38
public/app/features/explore/state/actions.ts
View File

@ -9,14 +9,14 @@ import {
DataQuery,
DataSourceApi,
dateTimeForTimeZone,
ExploreUrlState,
isDateTime,
LoadingState,
LogsDedupStrategy,
PanelData,
QueryFixAction,
RawTimeRange,
TimeRange,
ExploreUrlState,
LogsDedupStrategy,
} from '@grafana/data';
// Services & Utils
import store from 'app/core/store';
@ -40,17 +40,21 @@ import {
import {
addToRichHistory,
deleteAllFromRichHistory,
updateStarredInRichHistory,
updateCommentInRichHistory,
deleteQueryInRichHistory,
getRichHistory,
updateCommentInRichHistory,
updateStarredInRichHistory,
} from 'app/core/utils/richHistory';
// Types
import { ExploreItemState, ThunkResult } from 'app/types';
import { ThunkResult } from 'app/types';
import { ExploreId, QueryOptions } from 'app/types/explore';
import { ExploreId, ExploreItemState, QueryOptions } from 'app/types/explore';
import {
addQueryRowAction,
cancelQueriesAction,
changeDedupStrategyAction,
ChangeDedupStrategyPayload,
changeLoadingStateAction,
changeQueryAction,
changeRangeAction,
changeRefreshIntervalAction,
@ -59,7 +63,6 @@ import {
ChangeSizePayload,
clearQueriesAction,
historyUpdatedAction,
richHistoryUpdatedAction,
initializeExploreAction,
loadDatasourceMissingAction,
loadDatasourcePendingAction,
@ -69,6 +72,7 @@ import {
queriesImportedAction,
queryStoreSubscriptionAction,
queryStreamUpdatedAction,
richHistoryUpdatedAction,
scanStartAction,
scanStopAction,
setQueriesAction,
@ -77,19 +81,21 @@ import {
splitOpenAction,
syncTimesAction,
updateDatasourceInstanceAction,
changeLoadingStateAction,
cancelQueriesAction,
changeDedupStrategyAction,
ChangeDedupStrategyPayload,
} from './actionTypes';
import { getTimeZone } from 'app/features/profile/state/selectors';
import { getShiftedTimeRange } from 'app/core/utils/timePicker';
import { updateLocation } from '../../../core/actions';
import { getTimeSrv, TimeSrv } from '../../dashboard/services/TimeSrv';
import { preProcessPanelData, runRequest } from '../../dashboard/state/runRequest';
import { PanelModel, DashboardModel } from 'app/features/dashboard/state';
import { DashboardModel, PanelModel } from 'app/features/dashboard/state';
import { getExploreDatasources } from './selectors';
import { serializeStateToUrlParam } from '@grafana/data/src/utils/url';
import {
decorateWithGraphLogsTraceAndTable,
decorateWithGraphResult,
decorateWithLogsResult,
decorateWithTableResult,
} from '../utils/decorators';
/**
* Adds a query row after the row with the given index.
@ -466,9 +472,13 @@ export const runQueries = (exploreId: ExploreId): ThunkResult<void> => {
// rendering. In case this is optimized this can be tweaked, but also it should be only as fast as user
// actually can see what is happening.
live ? throttleTime(500) : identity,
map((data: PanelData) => preProcessPanelData(data, queryResponse))
map((data: PanelData) => preProcessPanelData(data, queryResponse)),
decorateWithGraphLogsTraceAndTable(getState().explore[exploreId].datasourceInstance),
decorateWithGraphResult(),
decorateWithTableResult(),
decorateWithLogsResult(getState().explore[exploreId])
)
.subscribe((data: PanelData) => {
.subscribe(data => {
if (!data.error && firstResponse) {
// Side-effect: Saving history in localstorage
const nextHistory = updateHistory(history, datasourceId, queries);

22
public/app/features/explore/state/reducers.ts
View File

@ -3,15 +3,14 @@ import { AnyAction } from 'redux';
import { PayloadAction } from '@reduxjs/toolkit';
import {
DataQuery,
DataQueryErrorType,
DefaultTimeRange,
LoadingState,
LogsDedupStrategy,
PanelData,
PanelEvents,
TimeZone,
toLegacyResponseData,
LogsDedupStrategy,
sortLogsResult,
DataQueryErrorType,
toLegacyResponseData,
} from '@grafana/data';
import { RefreshPicker } from '@grafana/ui';
import { LocationUpdate } from '@grafana/runtime';
@ -27,6 +26,8 @@ import {
import { ExploreId, ExploreItemState, ExploreState, ExploreUpdateState } from 'app/types/explore';
import {
addQueryRowAction,
cancelQueriesAction,
changeDedupStrategyAction,
changeLoadingStateAction,
changeQueryAction,
changeRangeAction,
@ -35,7 +36,6 @@ import {
clearQueriesAction,
highlightLogsExpressionAction,
historyUpdatedAction,
richHistoryUpdatedAction,
initializeExploreAction,
loadDatasourceMissingAction,
loadDatasourcePendingAction,
@ -48,6 +48,7 @@ import {
removeQueryRowAction,
resetExploreAction,
ResetExplorePayload,
richHistoryUpdatedAction,
scanStartAction,
scanStopAction,
setPausedStateAction,
@ -59,10 +60,7 @@ import {
syncTimesAction,
toggleLogLevelAction,
updateDatasourceInstanceAction,
cancelQueriesAction,
changeDedupStrategyAction,
} from './actionTypes';
import { ResultProcessor } from '../utils/ResultProcessor';
import { updateLocation } from '../../../core/actions';
import { Emitter } from 'app/core/core';
@ -464,7 +462,7 @@ export const processQueryResponse = (
action: PayloadAction<QueryEndedPayload>
): ExploreItemState => {
const { response } = action.payload;
const { request, state: loadingState, series, error } = response;
const { request, state: loadingState, series, error, graphResult, logsResult, tableResult, traceFrames } = response;
if (error) {
if (error.type === DataQueryErrorType.Timeout) {
@ -496,10 +494,6 @@ export const processQueryResponse = (
}
const latency = request.endTime ? request.endTime - request.startTime : 0;
const processor = new ResultProcessor(state, series, request.intervalMs, request.timezone as TimeZone);
const graphResult = processor.getGraphResult();
const tableResult = processor.getTableResult();
const logsResult = processor.getLogsResult();
// Send legacy data to Angular editors
if (state.datasourceInstance?.components?.QueryCtrl) {
@ -520,7 +514,7 @@ export const processQueryResponse = (
showLogs: !!logsResult,
showMetrics: !!graphResult,
showTable: !!tableResult,
showTrace: !!processor.traceFrames.length,
showTrace: !!traceFrames.length,
};
};

323
public/app/features/explore/utils/ResultProcessor.test.ts
View File

@ -1,323 +0,0 @@
jest.mock('@grafana/data/src/datetime/formatter', () => ({
dateTimeFormat: () => 'format() jest mocked',
dateTimeFormatTimeAgo: (ts: any) => 'fromNow() jest mocked',
}));
import { ResultProcessor } from './ResultProcessor';
import { ExploreItemState } from 'app/types/explore';
import TableModel from 'app/core/table_model';
import { FieldType, LogRowModel, TimeSeries, toDataFrame, ArrayVector } from '@grafana/data';
const testContext = (options: any = {}) => {
const timeSeries = toDataFrame({
name: 'A-series',
refId: 'A',
meta: {
preferredVisualisationType: 'graph',
},
fields: [
{ name: 'time', type: FieldType.time, values: [100, 200, 300] },
{ name: 'A-series', type: FieldType.number, values: [4, 5, 6] },
],
});
const table = toDataFrame({
name: 'table-res',
refId: 'A',
fields: [
{ name: 'value', type: FieldType.number, values: [4, 5, 6] },
{ name: 'time', type: FieldType.time, values: [100, 100, 100] },
{ name: 'tsNs', type: FieldType.time, values: ['100000002', undefined, '100000001'] },
{ name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] },
],
});
const emptyTable = toDataFrame({ name: 'empty-table', refId: 'A', fields: [] });
const logs = toDataFrame({
name: 'logs-res',
refId: 'A',
fields: [
{ name: 'value', type: FieldType.number, values: [4, 5, 6] },
{ name: 'time', type: FieldType.time, values: [100, 100, 100] },
{ name: 'tsNs', type: FieldType.time, values: ['100000002', undefined, '100000001'] },
{ name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] },
],
meta: { preferredVisualisationType: 'logs' },
});
const defaultOptions = {
dataFrames: [timeSeries, table, emptyTable, logs],
graphResult: [] as TimeSeries[],
tableResult: new TableModel(),
logsResult: { hasUniqueLabels: false, rows: [] as LogRowModel[] },
};
const combinedOptions = { ...defaultOptions, ...options };
const state = ({
graphResult: combinedOptions.graphResult,
tableResult: combinedOptions.tableResult,
logsResult: combinedOptions.logsResult,
queryIntervals: { intervalMs: 10 },
} as any) as ExploreItemState;
const resultProcessor = new ResultProcessor(state, combinedOptions.dataFrames, 60000, 'utc');
return {
dataFrames: combinedOptions.dataFrames,
resultProcessor,
};
};
describe('ResultProcessor', () => {
describe('constructed without result', () => {
describe('when calling getGraphResult', () => {
it('then it should return null', () => {
const { resultProcessor } = testContext({ dataFrames: [] });
const theResult = resultProcessor.getGraphResult();
expect(theResult).toEqual(null);
});
});
describe('when calling getTableResult', () => {
it('then it should return null', () => {
const { resultProcessor } = testContext({ dataFrames: [] });
const theResult = resultProcessor.getTableResult();
expect(theResult).toEqual(null);
});
});
describe('when calling getLogsResult', () => {
it('then it should return null', () => {
const { resultProcessor } = testContext({ dataFrames: [] });
const theResult = resultProcessor.getLogsResult();
expect(theResult).toBeNull();
});
});
});
describe('constructed with a result that is a DataQueryResponse', () => {
describe('when calling getGraphResult', () => {
it('then it should return correct graph result', () => {
const { resultProcessor, dataFrames } = testContext();
const timeField = dataFrames[0].fields[0];
const valueField = dataFrames[0].fields[1];
const theResult = resultProcessor.getGraphResult();
expect(theResult![0]).toEqual({
label: 'A-series',
color: '#7EB26D',
data: [
[100, 4],
[200, 5],
[300, 6],
],
info: [],
isVisible: true,
yAxis: {
index: 1,
},
seriesIndex: 0,
timeField,
valueField,
timeStep: 100,
});
});
});
describe('when calling getTableResult', () => {
it('then it should return correct table result', () => {
const { resultProcessor } = testContext();
let theResult = resultProcessor.getTableResult();
expect(theResult?.fields[0].name).toEqual('value');
expect(theResult?.fields[1].name).toEqual('time');
expect(theResult?.fields[2].name).toEqual('tsNs');
expect(theResult?.fields[3].name).toEqual('message');
expect(theResult?.fields[1].display).not.toBeNull();
expect(theResult?.length).toBe(3);
// Same data though a DataFrame
theResult = toDataFrame(
new TableModel({
columns: [
{ text: 'value', type: 'number' },
{ text: 'time', type: 'time' },
{ text: 'tsNs', type: 'time' },
{ text: 'message', type: 'string' },
],
rows: [
[4, 100, '100000000', 'this is a message'],
[5, 200, '100000000', 'second message'],
[6, 300, '100000000', 'third'],
],
type: 'table',
})
);
expect(theResult.fields[0].name).toEqual('value');
expect(theResult.fields[1].name).toEqual('time');
expect(theResult.fields[2].name).toEqual('tsNs');
expect(theResult.fields[3].name).toEqual('message');
expect(theResult.fields[1].display).not.toBeNull();
expect(theResult.length).toBe(3);
});
it('should do join transform if all series are timeseries', () => {
const { resultProcessor } = testContext({
dataFrames: [
toDataFrame({
name: 'A-series',
refId: 'A',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 200, 300] },
{ name: 'A-series', type: FieldType.number, values: [4, 5, 6] },
],
}),
toDataFrame({
name: 'B-series',
refId: 'B',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 200, 300] },
{ name: 'B-series', type: FieldType.number, values: [4, 5, 6] },
],
}),
],
});
let result = resultProcessor.getTableResult()!;
expect(result.fields[0].name).toBe('Time');
expect(result.fields[1].name).toBe('A-series');
expect(result.fields[2].name).toBe('B-series');
expect(result.fields[0].values.toArray()).toEqual([100, 200, 300]);
expect(result.fields[1].values.toArray()).toEqual([4, 5, 6]);
expect(result.fields[2].values.toArray()).toEqual([4, 5, 6]);
});
it('should not override fields display property when filled', () => {
const { resultProcessor, dataFrames } = testContext({
dataFrames: [
toDataFrame({
name: 'A-series',
refId: 'A',
fields: [{ name: 'Text', type: FieldType.string, values: ['someText'] }],
}),
],
});
const displayFunctionMock = jest.fn();
dataFrames[0].fields[0].display = displayFunctionMock;
const data = resultProcessor.getTableResult();
expect(data?.fields[0].display).toBe(displayFunctionMock);
});
});
describe('when calling getLogsResult', () => {
it('then it should return correct logs result', () => {
const { resultProcessor, dataFrames } = testContext({});
const logsDataFrame = dataFrames[3];
const theResult = resultProcessor.getLogsResult();
expect(theResult).toEqual({
hasUniqueLabels: false,
meta: [],
rows: [
{
rowIndex: 0,
dataFrame: logsDataFrame,
entry: 'this is a message',
entryFieldIndex: 3,
hasAnsi: false,
labels: {},
logLevel: 'unknown',
raw: 'this is a message',
searchWords: [] as string[],
timeEpochMs: 100,
timeEpochNs: '100000002',
timeFromNow: 'fromNow() jest mocked',
timeLocal: 'format() jest mocked',
timeUtc: 'format() jest mocked',
uid: '0',
uniqueLabels: {},
},
{
rowIndex: 2,
dataFrame: logsDataFrame,
entry: 'third',
entryFieldIndex: 3,
hasAnsi: false,
labels: {},
logLevel: 'unknown',
raw: 'third',
searchWords: [] as string[],
timeEpochMs: 100,
timeEpochNs: '100000001',
timeFromNow: 'fromNow() jest mocked',
timeLocal: 'format() jest mocked',
timeUtc: 'format() jest mocked',
uid: '2',
uniqueLabels: {},
},
{
rowIndex: 1,
dataFrame: logsDataFrame,
entry: 'second message',
entryFieldIndex: 3,
hasAnsi: false,
labels: {},
logLevel: 'unknown',
raw: 'second message',
searchWords: [] as string[],
timeEpochMs: 100,
timeEpochNs: '100000000',
timeFromNow: 'fromNow() jest mocked',
timeLocal: 'format() jest mocked',
timeUtc: 'format() jest mocked',
uid: '1',
uniqueLabels: {},
},
],
series: [
{
label: 'unknown',
color: '#8e8e8e',
data: [[0, 3]],
isVisible: true,
yAxis: {
index: 1,
min: 0,
tickDecimals: 0,
},
seriesIndex: 0,
timeField: {
name: 'Time',
type: 'time',
config: {},
values: new ArrayVector([0]),
index: 0,
display: expect.anything(),
},
valueField: {
name: 'unknown',
type: 'number',
config: { unit: undefined, color: '#8e8e8e' },
values: new ArrayVector([3]),
labels: undefined,
index: 1,
display: expect.anything(),
},
timeStep: 0,
},
],
visibleRange: undefined,
});
});
});
});
});

161
public/app/features/explore/utils/ResultProcessor.ts
View File

@ -1,161 +0,0 @@
import {
LogsModel,
GraphSeriesXY,
DataFrame,
FieldType,
TimeZone,
getDisplayProcessor,
PreferredVisualisationType,
standardTransformers,
sortLogsResult,
} from '@grafana/data';
import { ExploreItemState } from 'app/types/explore';
import { refreshIntervalToSortOrder } from 'app/core/utils/explore';
import { dataFrameToLogsModel } from 'app/core/logs_model';
import { getGraphSeriesModel } from 'app/plugins/panel/graph2/getGraphSeriesModel';
import { config } from 'app/core/config';
export class ResultProcessor {
graphFrames: DataFrame[] = [];
tableFrames: DataFrame[] = [];
logsFrames: DataFrame[] = [];
traceFrames: DataFrame[] = [];
constructor(
private state: ExploreItemState,
private dataFrames: DataFrame[],
private intervalMs: number,
private timeZone: TimeZone
) {
this.classifyFrames();
}
private classifyFrames() {
for (const frame of this.dataFrames) {
if (shouldShowInVisualisationTypeStrict(frame, 'logs')) {
this.logsFrames.push(frame);
} else if (shouldShowInVisualisationTypeStrict(frame, 'graph')) {
this.graphFrames.push(frame);
} else if (shouldShowInVisualisationTypeStrict(frame, 'trace')) {
this.traceFrames.push(frame);
} else if (shouldShowInVisualisationTypeStrict(frame, 'table')) {
this.tableFrames.push(frame);
} else if (isTimeSeries(frame, this.state.datasourceInstance?.meta.id)) {
if (shouldShowInVisualisationType(frame, 'graph')) {
this.graphFrames.push(frame);
}
if (shouldShowInVisualisationType(frame, 'table')) {
this.tableFrames.push(frame);
}
} else {
// We fallback to table if we do not have any better meta info about the dataframe.
this.tableFrames.push(frame);
}
}
}
getGraphResult(): GraphSeriesXY[] | null {
if (this.graphFrames.length === 0) {
return null;
}
return getGraphSeriesModel(
this.graphFrames,
this.timeZone,
{},
{ showBars: false, showLines: true, showPoints: false },
{ asTable: false, isVisible: true, placement: 'under' }
);
}
getTableResult(): DataFrame | null {
if (this.tableFrames.length === 0) {
return null;
}
this.tableFrames.sort((frameA: DataFrame, frameB: DataFrame) => {
const frameARefId = frameA.refId!;
const frameBRefId = frameB.refId!;
if (frameARefId > frameBRefId) {
return 1;
}
if (frameARefId < frameBRefId) {
return -1;
}
return 0;
});
const hasOnlyTimeseries = this.tableFrames.every(df => isTimeSeries(df));
// If we have only timeseries we do join on default time column which makes more sense. If we are showing
// non timeseries or some mix of data we are not trying to join on anything and just try to merge them in
// single table, which may not make sense in most cases, but it's up to the user to query something sensible.
const transformer = hasOnlyTimeseries
? standardTransformers.seriesToColumnsTransformer.transformer({})
: standardTransformers.mergeTransformer.transformer({});
const data = transformer(this.tableFrames)[0];
// set display processor
for (const field of data.fields) {
field.display =
field.display ??
getDisplayProcessor({
field,
theme: config.theme,
timeZone: this.timeZone,
});
}
return data;
}
getLogsResult(): LogsModel | null {
if (this.logsFrames.length === 0) {
return null;
}
const newResults = dataFrameToLogsModel(this.logsFrames, this.intervalMs, this.timeZone, this.state.absoluteRange);
const sortOrder = refreshIntervalToSortOrder(this.state.refreshInterval);
const sortedNewResults = sortLogsResult(newResults, sortOrder);
const rows = sortedNewResults.rows;
const series = sortedNewResults.series;
return { ...sortedNewResults, rows, series };
}
}
function isTimeSeries(frame: DataFrame, datasource?: string): boolean {
// TEMP: Temporary hack. Remove when logs/metrics unification is done
if (datasource && datasource === 'cloudwatch') {
return isTimeSeriesCloudWatch(frame);
}
if (frame.fields.length === 2) {
if (frame.fields[0].type === FieldType.time) {
return true;
}
}
return false;
}
function shouldShowInVisualisationType(frame: DataFrame, visualisation: PreferredVisualisationType) {
if (frame.meta?.preferredVisualisationType && frame.meta?.preferredVisualisationType !== visualisation) {
return false;
}
return true;
}
function shouldShowInVisualisationTypeStrict(frame: DataFrame, visualisation: PreferredVisualisationType) {
return frame.meta?.preferredVisualisationType === visualisation;
}
// TEMP: Temporary hack. Remove when logs/metrics unification is done
function isTimeSeriesCloudWatch(frame: DataFrame): boolean {
return (
frame.fields.some(field => field.type === FieldType.time) &&
frame.fields.some(field => field.type === FieldType.number)
);
}

534
public/app/features/explore/utils/decorators.test.ts Normal file
View File

@ -0,0 +1,534 @@
jest.mock('@grafana/data/src/datetime/formatter', () => ({
dateTimeFormat: () => 'format() jest mocked',
dateTimeFormatTimeAgo: (ts: any) => 'fromNow() jest mocked',
}));
import { of } from 'rxjs';
import {
ArrayVector,
DataFrame,
DataQueryRequest,
DataSourceApi,
FieldType,
LoadingState,
observableTester,
PanelData,
TimeRange,
toDataFrame,
} from '@grafana/data';
import {
decorateWithGraphLogsTraceAndTable,
decorateWithGraphResult,
decorateWithLogsResult,
decorateWithTableResult,
} from './decorators';
import { describe } from '../../../../test/lib/common';
import { ExploreItemState, ExplorePanelData } from 'app/types';
import TableModel from 'app/core/table_model';
const getTestContext = () => {
const timeSeries = toDataFrame({
name: 'A-series',
refId: 'A',
meta: {
preferredVisualisationType: 'graph',
},
fields: [
{ name: 'time', type: FieldType.time, values: [100, 200, 300] },
{ name: 'A-series', type: FieldType.number, values: [4, 5, 6] },
],
});
const table = toDataFrame({
name: 'table-res',
refId: 'A',
fields: [
{ name: 'value', type: FieldType.number, values: [4, 5, 6] },
{ name: 'time', type: FieldType.time, values: [100, 100, 100] },
{ name: 'tsNs', type: FieldType.time, values: ['100000002', undefined, '100000001'] },
{ name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] },
],
});
const emptyTable = toDataFrame({ name: 'empty-table', refId: 'A', fields: [] });
const logs = toDataFrame({
name: 'logs-res',
refId: 'A',
fields: [
{ name: 'value', type: FieldType.number, values: [4, 5, 6] },
{ name: 'time', type: FieldType.time, values: [100, 100, 100] },
{ name: 'tsNs', type: FieldType.time, values: ['100000002', undefined, '100000001'] },
{ name: 'message', type: FieldType.string, values: ['this is a message', 'second message', 'third'] },
],
meta: { preferredVisualisationType: 'logs' },
});
return { emptyTable, timeSeries, logs, table };
};
const createExplorePanelData = (args: Partial<ExplorePanelData>): ExplorePanelData => {
const defaults: ExplorePanelData = {
series: [],
timeRange: ({} as unknown) as TimeRange,
state: LoadingState.Done,
graphFrames: [],
graphResult: (undefined as unknown) as null,
logsFrames: [],
logsResult: (undefined as unknown) as null,
tableFrames: [],
tableResult: (undefined as unknown) as null,
traceFrames: [],
};
return { ...defaults, ...args };
};
describe('decorateWithGraphLogsTraceAndTable', () => {
describe('when used without error', () => {
it('then the result should be correct', done => {
const { table, logs, timeSeries, emptyTable } = getTestContext();
const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi;
const series = [table, logs, timeSeries, emptyTable];
const panelData: PanelData = {
series,
state: LoadingState.Done,
timeRange: ({} as unknown) as TimeRange,
};
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)),
expect: value => {
expect(value).toEqual({
series,
state: LoadingState.Done,
timeRange: {},
graphFrames: [timeSeries],
tableFrames: [table, emptyTable],
logsFrames: [logs],
traceFrames: [],
graphResult: null,
tableResult: null,
logsResult: null,
});
},
done,
});
});
});
describe('when used without frames', () => {
it('then the result should be correct', done => {
const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi;
const series: DataFrame[] = [];
const panelData: PanelData = {
series,
state: LoadingState.Done,
timeRange: ({} as unknown) as TimeRange,
};
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)),
expect: value => {
expect(value).toEqual({
series: [],
state: LoadingState.Done,
timeRange: {},
graphFrames: [],
tableFrames: [],
logsFrames: [],
traceFrames: [],
graphResult: null,
tableResult: null,
logsResult: null,
});
},
done,
});
});
});
describe('when used with an error', () => {
it('then the result should be correct', done => {
const { timeSeries, logs, table } = getTestContext();
const datasourceInstance = ({ meta: { id: 'prometheus' } } as unknown) as DataSourceApi;
const series: DataFrame[] = [timeSeries, logs, table];
const panelData: PanelData = {
series,
error: {},
state: LoadingState.Error,
timeRange: ({} as unknown) as TimeRange,
};
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphLogsTraceAndTable(datasourceInstance)),
expect: value => {
expect(value).toEqual({
series: [timeSeries, logs, table],
error: {},
state: LoadingState.Error,
timeRange: {},
graphFrames: [],
tableFrames: [],
logsFrames: [],
traceFrames: [],
graphResult: null,
tableResult: null,
logsResult: null,
});
},
done,
});
});
});
});
describe('decorateWithGraphResult', () => {
describe('when used without error', () => {
it('then the graphResult should be correct', done => {
const { timeSeries } = getTestContext();
const timeField = timeSeries.fields[0];
const valueField = timeSeries.fields[1];
const panelData = createExplorePanelData({ graphFrames: [timeSeries] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphResult()),
expect: panelData => {
expect(panelData.graphResult![0]).toEqual({
label: 'A-series',
color: '#7EB26D',
data: [
[100, 4],
[200, 5],
[300, 6],
],
info: [],
isVisible: true,
yAxis: {
index: 1,
},
seriesIndex: 0,
timeField,
valueField,
timeStep: 100,
});
},
done,
});
});
});
describe('when used without error but graph frames are empty', () => {
it('then the graphResult should be null', done => {
const panelData = createExplorePanelData({ graphFrames: [] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphResult()),
expect: panelData => {
expect(panelData.graphResult).toBeNull();
},
done,
});
});
});
describe('when used with error', () => {
it('then the graphResult should be null', done => {
const { timeSeries } = getTestContext();
const panelData = createExplorePanelData({ error: {}, graphFrames: [timeSeries] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithGraphResult()),
expect: panelData => {
expect(panelData.graphResult).toBeNull();
},
done,
});
});
});
});
describe('decorateWithTableResult', () => {
describe('when used without error', () => {
it('then the tableResult should be correct', done => {
const { table, emptyTable } = getTestContext();
const panelData = createExplorePanelData({ tableFrames: [table, emptyTable] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
let theResult = panelData.tableResult;
expect(theResult?.fields[0].name).toEqual('value');
expect(theResult?.fields[1].name).toEqual('time');
expect(theResult?.fields[2].name).toEqual('tsNs');
expect(theResult?.fields[3].name).toEqual('message');
expect(theResult?.fields[1].display).not.toBeNull();
expect(theResult?.length).toBe(3);
// I don't understand the purpose of the code below, feels like this belongs in toDataFrame tests?
// Same data though a DataFrame
theResult = toDataFrame(
new TableModel({
columns: [
{ text: 'value', type: 'number' },
{ text: 'time', type: 'time' },
{ text: 'tsNs', type: 'time' },
{ text: 'message', type: 'string' },
],
rows: [
[4, 100, '100000000', 'this is a message'],
[5, 200, '100000000', 'second message'],
[6, 300, '100000000', 'third'],
],
type: 'table',
})
);
expect(theResult.fields[0].name).toEqual('value');
expect(theResult.fields[1].name).toEqual('time');
expect(theResult.fields[2].name).toEqual('tsNs');
expect(theResult.fields[3].name).toEqual('message');
expect(theResult.fields[1].display).not.toBeNull();
expect(theResult.length).toBe(3);
},
done,
});
});
it('should do join transform if all series are timeseries', done => {
const tableFrames = [
toDataFrame({
name: 'A-series',
refId: 'A',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 200, 300] },
{ name: 'A-series', type: FieldType.number, values: [4, 5, 6] },
],
}),
toDataFrame({
name: 'B-series',
refId: 'B',
fields: [
{ name: 'Time', type: FieldType.time, values: [100, 200, 300] },
{ name: 'B-series', type: FieldType.number, values: [4, 5, 6] },
],
}),
];
const panelData = createExplorePanelData({ tableFrames });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
const result = panelData.tableResult;
expect(result?.fields[0].name).toBe('Time');
expect(result?.fields[1].name).toBe('A-series');
expect(result?.fields[2].name).toBe('B-series');
expect(result?.fields[0].values.toArray()).toEqual([100, 200, 300]);
expect(result?.fields[1].values.toArray()).toEqual([4, 5, 6]);
expect(result?.fields[2].values.toArray()).toEqual([4, 5, 6]);
},
done,
});
});
it('should not override fields display property when filled', done => {
const tableFrames = [
toDataFrame({
name: 'A-series',
refId: 'A',
fields: [{ name: 'Text', type: FieldType.string, values: ['someText'] }],
}),
];
const displayFunctionMock = jest.fn();
tableFrames[0].fields[0].display = displayFunctionMock;
const panelData = createExplorePanelData({ tableFrames });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
const data = panelData.tableResult;
expect(data?.fields[0].display).toBe(displayFunctionMock);
},
done,
});
});
});
describe('when used without error but table frames are empty', () => {
it('then the tableResult should be null', done => {
const panelData = createExplorePanelData({ tableFrames: [] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
expect(panelData.tableResult).toBeNull();
},
done,
});
});
});
describe('when used with error', () => {
it('then the tableResult should be null', done => {
const { table, emptyTable } = getTestContext();
const panelData = createExplorePanelData({ error: {}, tableFrames: [table, emptyTable] });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithTableResult()),
expect: panelData => {
expect(panelData.tableResult).toBeNull();
},
done,
});
});
});
});
describe('decorateWithLogsResult', () => {
describe('when used without error', () => {
it('then the logsResult should be correct', done => {
const { logs } = getTestContext();
const state = ({
queryIntervals: { intervalMs: 10 },
} as unknown) as ExploreItemState;
const request = ({ timezone: 'utc', intervalMs: 60000 } as unknown) as DataQueryRequest;
const panelData = createExplorePanelData({ logsFrames: [logs], request });
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithLogsResult(state)),
expect: panelData => {
const theResult = panelData.logsResult;
expect(theResult).toEqual({
hasUniqueLabels: false,
meta: [],
rows: [
{
rowIndex: 0,
dataFrame: logs,
entry: 'this is a message',
entryFieldIndex: 3,
hasAnsi: false,
labels: {},
logLevel: 'unknown',
raw: 'this is a message',
searchWords: [] as string[],
timeEpochMs: 100,
timeEpochNs: '100000002',
timeFromNow: 'fromNow() jest mocked',
timeLocal: 'format() jest mocked',
timeUtc: 'format() jest mocked',
uid: '0',
uniqueLabels: {},
},
{
rowIndex: 2,
dataFrame: logs,
entry: 'third',
entryFieldIndex: 3,
hasAnsi: false,
labels: {},
logLevel: 'unknown',
raw: 'third',
searchWords: [] as string[],
timeEpochMs: 100,
timeEpochNs: '100000001',
timeFromNow: 'fromNow() jest mocked',
timeLocal: 'format() jest mocked',
timeUtc: 'format() jest mocked',
uid: '2',
uniqueLabels: {},
},
{
rowIndex: 1,
dataFrame: logs,
entry: 'second message',
entryFieldIndex: 3,
hasAnsi: false,
labels: {},
logLevel: 'unknown',
raw: 'second message',
searchWords: [] as string[],
timeEpochMs: 100,
timeEpochNs: '100000000',
timeFromNow: 'fromNow() jest mocked',
timeLocal: 'format() jest mocked',
timeUtc: 'format() jest mocked',
uid: '1',
uniqueLabels: {},
},
],
series: [
{
label: 'unknown',
color: '#8e8e8e',
data: [[0, 3]],
isVisible: true,
yAxis: {
index: 1,
min: 0,
tickDecimals: 0,
},
seriesIndex: 0,
timeField: {
name: 'Time',
type: 'time',
config: {},
values: new ArrayVector([0]),
index: 0,
display: expect.anything(),
},
valueField: {
name: 'unknown',
type: 'number',
config: { unit: undefined, color: '#8e8e8e' },
values: new ArrayVector([3]),
labels: undefined,
index: 1,
display: expect.anything(),
},
timeStep: 0,
},
],
visibleRange: undefined,
});
},
done,
});
});
});
describe('when used without error but logs frames are empty', () => {
it('then the graphResult should be null', done => {
const panelData = createExplorePanelData({ logsFrames: [] });
const state = ({} as unknown) as ExploreItemState;
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithLogsResult(state)),
expect: panelData => {
expect(panelData.logsResult).toBeNull();
},
done,
});
});
});
describe('when used with error', () => {
it('then the graphResult should be null', done => {
const { logs } = getTestContext();
const panelData = createExplorePanelData({ error: {}, logsFrames: [logs] });
const state = ({} as unknown) as ExploreItemState;
observableTester().subscribeAndExpectOnNext({
observable: of(panelData).pipe(decorateWithLogsResult(state)),
expect: panelData => {
expect(panelData.logsResult).toBeNull();
},
done,
});
});
});
});

214
public/app/features/explore/utils/decorators.ts Normal file
View File

@ -0,0 +1,214 @@
import { MonoTypeOperatorFunction, of, OperatorFunction } from 'rxjs';
import { map, mergeMap } from 'rxjs/operators';
import {
DataFrame,
DataSourceApi,
FieldType,
getDisplayProcessor,
PanelData,
PreferredVisualisationType,
sortLogsResult,
standardTransformers,
} from '@grafana/data';
import { config } from '@grafana/runtime';
import { ExploreItemState, ExplorePanelData } from '../../../types';
import { getGraphSeriesModel } from '../../../plugins/panel/graph2/getGraphSeriesModel';
import { dataFrameToLogsModel } from '../../../core/logs_model';
import { refreshIntervalToSortOrder } from '../../../core/utils/explore';
export const decorateWithGraphLogsTraceAndTable = (
datasourceInstance?: DataSourceApi | null
): OperatorFunction<PanelData, ExplorePanelData> => inputStream =>
inputStream.pipe(
map(data => {
if (data.error) {
return {
...data,
graphFrames: [],
tableFrames: [],
logsFrames: [],
traceFrames: [],
graphResult: null,
tableResult: null,
logsResult: null,
};
}
const graphFrames: DataFrame[] = [];
const tableFrames: DataFrame[] = [];
const logsFrames: DataFrame[] = [];
const traceFrames: DataFrame[] = [];
for (const frame of data.series) {
if (shouldShowInVisualisationTypeStrict(frame, 'logs')) {
logsFrames.push(frame);
} else if (shouldShowInVisualisationTypeStrict(frame, 'graph')) {
graphFrames.push(frame);
} else if (shouldShowInVisualisationTypeStrict(frame, 'trace')) {
traceFrames.push(frame);
} else if (shouldShowInVisualisationTypeStrict(frame, 'table')) {
tableFrames.push(frame);
} else if (isTimeSeries(frame, datasourceInstance?.meta.id)) {
if (shouldShowInVisualisationType(frame, 'graph')) {
graphFrames.push(frame);
}
if (shouldShowInVisualisationType(frame, 'table')) {
tableFrames.push(frame);
}
} else {
// We fallback to table if we do not have any better meta info about the dataframe.
tableFrames.push(frame);
}
}
return {
...data,
graphFrames,
tableFrames,
logsFrames,
traceFrames,
graphResult: null,
tableResult: null,
logsResult: null,
};
})
);
export const decorateWithGraphResult = (): MonoTypeOperatorFunction<ExplorePanelData> => inputStream =>
inputStream.pipe(
map(data => {
if (data.error) {
return { ...data, graphResult: null };
}
const graphResult =
data.graphFrames.length === 0
? null
: getGraphSeriesModel(
data.graphFrames,
data.request?.timezone ?? 'browser',
{},
{ showBars: false, showLines: true, showPoints: false },
{ asTable: false, isVisible: true, placement: 'under' }
);
return { ...data, graphResult };
})
);
export const decorateWithTableResult = (): MonoTypeOperatorFunction<ExplorePanelData> => inputStream =>
inputStream.pipe(
mergeMap(data => {
if (data.error) {
return of({ ...data, tableResult: null });
}
if (data.tableFrames.length === 0) {
return of({ ...data, tableResult: null });
}
data.tableFrames.sort((frameA: DataFrame, frameB: DataFrame) => {
const frameARefId = frameA.refId!;
const frameBRefId = frameB.refId!;
if (frameARefId > frameBRefId) {
return 1;
}
if (frameARefId < frameBRefId) {
return -1;
}
return 0;
});
const hasOnlyTimeseries = data.tableFrames.every(df => isTimeSeries(df));
// If we have only timeseries we do join on default time column which makes more sense. If we are showing
// non timeseries or some mix of data we are not trying to join on anything and just try to merge them in
// single table, which may not make sense in most cases, but it's up to the user to query something sensible.
const transformer = hasOnlyTimeseries
? of(data.tableFrames).pipe(standardTransformers.seriesToColumnsTransformer.operator({}))
: of(data.tableFrames).pipe(standardTransformers.mergeTransformer.operator({}));
return transformer.pipe(
map(frames => {
const frame = frames[0];
// set display processor
for (const field of frame.fields) {
field.display =
field.display ??
getDisplayProcessor({
field,
theme: config.theme,
timeZone: data.request?.timezone ?? 'browser',
});
}
return { ...data, tableResult: frame };
})
);
})
);
export const decorateWithLogsResult = (
state: ExploreItemState
): MonoTypeOperatorFunction<ExplorePanelData> => inputStream =>
inputStream.pipe(
map(data => {
if (data.error) {
return { ...data, logsResult: null };
}
const { absoluteRange, refreshInterval } = state;
if (data.logsFrames.length === 0) {
return { ...data, logsResult: null };
}
const timeZone = data.request?.timezone ?? 'browser';
const intervalMs = data.request?.intervalMs;
const newResults = dataFrameToLogsModel(data.logsFrames, intervalMs, timeZone, absoluteRange);
const sortOrder = refreshIntervalToSortOrder(refreshInterval);
const sortedNewResults = sortLogsResult(newResults, sortOrder);
const rows = sortedNewResults.rows;
const series = sortedNewResults.series;
const logsResult = { ...sortedNewResults, rows, series };
return { ...data, logsResult };
})
);
function isTimeSeries(frame: DataFrame, datasource?: string): boolean {
// TEMP: Temporary hack. Remove when logs/metrics unification is done
if (datasource && datasource === 'cloudwatch') {
return isTimeSeriesCloudWatch(frame);
}
if (frame.fields.length === 2) {
if (frame.fields[0].type === FieldType.time) {
return true;
}
}
return false;
}
function shouldShowInVisualisationType(frame: DataFrame, visualisation: PreferredVisualisationType) {
if (frame.meta?.preferredVisualisationType && frame.meta?.preferredVisualisationType !== visualisation) {
return false;
}
return true;
}
function shouldShowInVisualisationTypeStrict(frame: DataFrame, visualisation: PreferredVisualisationType) {
return frame.meta?.preferredVisualisationType === visualisation;
}
// TEMP: Temporary hack. Remove when logs/metrics unification is done
function isTimeSeriesCloudWatch(frame: DataFrame): boolean {
return (
frame.fields.some(field => field.type === FieldType.time) &&
frame.fields.some(field => field.type === FieldType.number)
);
}

2
public/app/plugins/datasource/loki/datasource.test.ts
View File

@ -8,6 +8,7 @@ import {
DataQueryResponse,
dateTime,
FieldCache,
observableTester,
TimeRange,
} from '@grafana/data';
import { BackendSrvRequest, FetchResponse } from '@grafana/runtime';
@ -19,7 +20,6 @@ import { TemplateSrv } from 'app/features/templating/template_srv';
import { backendSrv } from 'app/core/services/backend_srv';
import { CustomVariableModel } from '../../../features/variables/types';
import { initialCustomVariableModelState } from '../../../features/variables/custom/reducer';
import { observableTester } from '../../../../test/helpers/observableTester';
import { expect } from '../../../../test/lib/common';
import { makeMockLokiDatasource } from './mocks';

3
public/app/plugins/datasource/postgres/specs/datasource.test.ts
View File

@ -1,14 +1,13 @@
import { of } from 'rxjs';
import { TestScheduler } from 'rxjs/testing';
import { FetchResponse } from '@grafana/runtime';
import { dateTime, toUtc } from '@grafana/data';
import { dateTime, observableTester, toUtc } from '@grafana/data';
import { PostgresDatasource } from '../datasource';
import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
import { TemplateSrv } from 'app/features/templating/template_srv';
import { initialCustomVariableModelState } from '../../../../features/variables/custom/reducer';
import { TimeSrv } from '../../../../features/dashboard/services/TimeSrv';
import { observableTester } from '../../../../../test/helpers/observableTester';
jest.mock('@grafana/runtime', () => ({
...((jest.requireActual('@grafana/runtime') as unknown) as object),

18
public/app/plugins/panel/graph3/GraphPanel.tsx
View File

@ -1,4 +1,4 @@
import React, { useMemo } from 'react';
import React, { useEffect, useState } from 'react';
import {
Area,
Canvas,
@ -9,14 +9,15 @@ import {
LegendPlugin,
Line,
Point,
SeriesGeometry,
Scale,
SeriesGeometry,
TooltipPlugin,
UPlotChart,
ZoomPlugin,
} from '@grafana/ui';
import {
DataFrame,
FieldConfig,
FieldType,
formattedValueToString,
@ -91,11 +92,18 @@ export const GraphPanel: React.FC<GraphPanelProps> = ({
options,
onChangeTimeRange,
}) => {
const alignedData = useMemo(() => {
const [alignedData, setAlignedData] = useState<DataFrame | null>(null);
useEffect(() => {
if (!data || !data.series?.length) {
return null;
setAlignedData(null);
return;
}
return alignAndSortDataFramesByFieldName(data.series, TIME_FIELD_NAME);
const subscription = alignAndSortDataFramesByFieldName(data.series, TIME_FIELD_NAME).subscribe(setAlignedData);
return function unsubscribe() {
subscription.unsubscribe();
};
}, [data]);
if (!alignedData) {

17
public/app/plugins/panel/graph3/utils.ts
View File

@ -1,7 +1,9 @@
import { Observable } from 'rxjs';
import { DataFrame, FieldType, getTimeField, sortDataFrame, transformDataFrame } from '@grafana/data';
import { map } from 'rxjs/operators';
// very time oriented for now
export const alignAndSortDataFramesByFieldName = (data: DataFrame[], fieldName: string) => {
export const alignAndSortDataFramesByFieldName = (data: DataFrame[], fieldName: string): Observable<DataFrame> => {
// normalize time field names
// in each frame find first time field and rename it to unified name
for (let i = 0; i < data.length; i++) {
@ -24,7 +26,7 @@ export const alignAndSortDataFramesByFieldName = (data: DataFrame[], fieldName:
// uPlot data needs to be aligned on x-axis (ref. https://github.com/leeoniya/uPlot/issues/108)
// For experimentation just assuming alignment on time field, needs to change
const aligned = transformDataFrame(
return transformDataFrame(
[
{
id: 'seriesToColumns',
@ -32,8 +34,11 @@ export const alignAndSortDataFramesByFieldName = (data: DataFrame[], fieldName:
},
],
dataFramesToPlot
)[0];
// need to be more "clever", not time only in the future!
return sortDataFrame(aligned, getTimeField(aligned).timeIndex!);
).pipe(
map(data => {
const aligned = data[0];
// need to be more "clever", not time only in the future!
return sortDataFrame(aligned, getTimeField(aligned).timeIndex!);
})
);
};

34
public/app/types/explore.ts
View File

@ -1,20 +1,20 @@
import { Unsubscribable } from 'rxjs';
import {
HistoryItem,
DataQuery,
DataSourceApi,
QueryHint,
PanelData,
DataQueryRequest,
RawTimeRange,
LogLevel,
TimeRange,
LogsModel,
LogsDedupStrategy,
AbsoluteTimeRange,
GraphSeriesXY,
DataFrame,
DataQuery,
DataQueryRequest,
DataSourceApi,
ExploreUrlState,
GraphSeriesXY,
HistoryItem,
LogLevel,
LogsDedupStrategy,
LogsModel,
PanelData,
QueryHint,
RawTimeRange,
TimeRange,
} from '@grafana/data';
import { Emitter } from 'app/core/core';
@ -219,3 +219,13 @@ export type RichHistoryQuery = {
sessionName: string;
timeRange?: string;
};
export interface ExplorePanelData extends PanelData {
graphFrames: DataFrame[];
tableFrames: DataFrame[];
logsFrames: DataFrame[];
traceFrames: DataFrame[];
graphResult: GraphSeriesXY[] | null;
tableResult: DataFrame | null;
logsResult: LogsModel | null;
}