mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Transformations: improve the reduce transformer (#27875)
This commit is contained in:
parent
db071e4939
commit
f8e0adb168
@ -1,4 +1,4 @@
|
||||
import { Field, DataFrame } from '../../types/dataFrame';
|
||||
import { Field, DataFrame, FieldType } from '../../types/dataFrame';
|
||||
import { MatcherID } from './ids';
|
||||
import { getFieldMatcher, fieldMatchers, getFrameMatchers, frameMatchers } from '../matchers';
|
||||
import { FieldMatcherInfo, MatcherConfig, FrameMatcherInfo } from '../../types/transformations';
|
||||
@ -191,6 +191,10 @@ export const neverFieldMatcher = (field: Field) => {
|
||||
return false;
|
||||
};
|
||||
|
||||
export const notTimeFieldMatcher = (field: Field) => {
|
||||
return field.type !== FieldType.time;
|
||||
};
|
||||
|
||||
export const neverFrameMatcher = (frame: DataFrame) => {
|
||||
return false;
|
||||
};
|
||||
|
@ -2,11 +2,13 @@ import { ReducerID } from '../fieldReducer';
|
||||
import { DataTransformerID } from './ids';
|
||||
import { toDataFrame } from '../../dataframe/processDataFrame';
|
||||
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
|
||||
import { reduceTransformer } from './reduce';
|
||||
import { reduceFields, reduceTransformer } from './reduce';
|
||||
import { transformDataFrame } from '../transformDataFrame';
|
||||
import { Field, FieldType } from '../../types';
|
||||
import { ArrayVector } from '../../vector';
|
||||
import { observableTester } from '../../utils/tests/observableTester';
|
||||
import { notTimeFieldMatcher } from '../matchers/predicates';
|
||||
import { DataFrameView } from '../../dataframe';
|
||||
|
||||
const seriesAWithSingleField = toDataFrame({
|
||||
name: 'A',
|
||||
@ -254,4 +256,31 @@ describe('Reducer Transformer', () => {
|
||||
done,
|
||||
});
|
||||
});
|
||||
|
||||
it('reduces fields with single calculator', () => {
|
||||
const frames = reduceFields(
|
||||
[seriesAWithSingleField, seriesAWithMultipleFields], // data
|
||||
notTimeFieldMatcher, // skip time fields
|
||||
[ReducerID.last] // only one
|
||||
);
|
||||
|
||||
// Convert each frame to a structure with the same fields
|
||||
expect(frames.length).toEqual(2);
|
||||
expect(frames[0].length).toEqual(1);
|
||||
expect(frames[1].length).toEqual(1);
|
||||
|
||||
const view0 = new DataFrameView<any>(frames[0]);
|
||||
const view1 = new DataFrameView<any>(frames[1]);
|
||||
expect({ ...view0.get(0) }).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"temperature": 6,
|
||||
}
|
||||
`);
|
||||
expect({ ...view1.get(0) }).toMatchInlineSnapshot(`
|
||||
Object {
|
||||
"humidity": 10000.6,
|
||||
"temperature": 6,
|
||||
}
|
||||
`);
|
||||
});
|
||||
});
|
||||
|
@ -3,19 +3,24 @@ import { map } from 'rxjs/operators';
|
||||
import { DataTransformerID } from './ids';
|
||||
import { DataTransformerInfo, MatcherConfig } from '../../types/transformations';
|
||||
import { fieldReducers, reduceField, ReducerID } from '../fieldReducer';
|
||||
import { alwaysFieldMatcher } from '../matchers/predicates';
|
||||
import { alwaysFieldMatcher, notTimeFieldMatcher } from '../matchers/predicates';
|
||||
import { DataFrame, Field, FieldType } from '../../types/dataFrame';
|
||||
import { ArrayVector } from '../../vector/ArrayVector';
|
||||
import { KeyValue } from '../../types/data';
|
||||
import { guessFieldTypeForField } from '../../dataframe/processDataFrame';
|
||||
import { getFieldMatcher } from '../matchers';
|
||||
import { FieldMatcherID } from '../matchers/ids';
|
||||
import { filterFieldsTransformer } from './filter';
|
||||
import { getFieldDisplayName } from '../../field';
|
||||
import { FieldMatcher } from '../../types/transformations';
|
||||
|
||||
export enum ReduceTransformerMode {
|
||||
SeriesToRows = 'seriesToRows', // default
|
||||
ReduceFields = 'reduceFields', // same structure, add additional row for each type
|
||||
}
|
||||
export interface ReduceTransformerOptions {
|
||||
reducers: ReducerID[];
|
||||
fields?: MatcherConfig; // Assume all fields
|
||||
mode?: ReduceTransformerMode;
|
||||
includeTimeField?: boolean;
|
||||
}
|
||||
|
||||
export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> = {
|
||||
@ -33,89 +38,111 @@ export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> =
|
||||
operator: options => source =>
|
||||
source.pipe(
|
||||
map(data => {
|
||||
const matcher = options.fields ? getFieldMatcher(options.fields) : alwaysFieldMatcher;
|
||||
const calculators = options.reducers && options.reducers.length ? fieldReducers.list(options.reducers) : [];
|
||||
const reducers = calculators.map(c => c.id);
|
||||
|
||||
const processed: DataFrame[] = [];
|
||||
|
||||
for (let seriesIndex = 0; seriesIndex < data.length; seriesIndex++) {
|
||||
const series = data[seriesIndex];
|
||||
const values: ArrayVector[] = [];
|
||||
const fields: Field[] = [];
|
||||
const byId: KeyValue<ArrayVector> = {};
|
||||
|
||||
values.push(new ArrayVector()); // The name
|
||||
fields.push({
|
||||
name: 'Field',
|
||||
type: FieldType.string,
|
||||
values: values[0],
|
||||
config: {},
|
||||
});
|
||||
|
||||
for (const info of calculators) {
|
||||
const vals = new ArrayVector();
|
||||
byId[info.id] = vals;
|
||||
values.push(vals);
|
||||
|
||||
fields.push({
|
||||
name: info.name,
|
||||
type: FieldType.other, // UNKNOWN until after we call the functions
|
||||
values: values[values.length - 1],
|
||||
config: {},
|
||||
});
|
||||
}
|
||||
|
||||
for (let i = 0; i < series.fields.length; i++) {
|
||||
const field = series.fields[i];
|
||||
|
||||
if (field.type === FieldType.time) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (matcher(field, series, data)) {
|
||||
const results = reduceField({
|
||||
field,
|
||||
reducers,
|
||||
});
|
||||
|
||||
// Update the name list
|
||||
const fieldName = getFieldDisplayName(field, series, data);
|
||||
|
||||
values[0].buffer.push(fieldName);
|
||||
|
||||
for (const info of calculators) {
|
||||
const v = results[info.id];
|
||||
byId[info.id].buffer.push(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const f of fields) {
|
||||
const t = guessFieldTypeForField(f);
|
||||
|
||||
if (t) {
|
||||
f.type = t;
|
||||
}
|
||||
}
|
||||
|
||||
processed.push({
|
||||
...series, // Same properties, different fields
|
||||
fields,
|
||||
length: values[0].length,
|
||||
});
|
||||
if (!options?.reducers?.length) {
|
||||
return data; // nothing selected
|
||||
}
|
||||
|
||||
return processed;
|
||||
}),
|
||||
filterFieldsTransformer.operator({ exclude: { id: FieldMatcherID.time } }),
|
||||
map(mergeResults)
|
||||
const matcher = options.fields
|
||||
? getFieldMatcher(options.fields)
|
||||
: options.includeTimeField && options.mode === ReduceTransformerMode.ReduceFields
|
||||
? alwaysFieldMatcher
|
||||
: notTimeFieldMatcher;
|
||||
|
||||
// Collapse all matching fields into a single row
|
||||
if (options.mode === ReduceTransformerMode.ReduceFields) {
|
||||
return reduceFields(data, matcher, options.reducers);
|
||||
}
|
||||
|
||||
// Add a row for each series
|
||||
const res = reduceSeriesToRows(data, matcher, options.reducers);
|
||||
return res ? [res] : [];
|
||||
})
|
||||
),
|
||||
};
|
||||
|
||||
const mergeResults = (data: DataFrame[]) => {
|
||||
if (data.length <= 1) {
|
||||
return data;
|
||||
/**
|
||||
* @internal only exported for testing
|
||||
*/
|
||||
export function reduceSeriesToRows(
|
||||
data: DataFrame[],
|
||||
matcher: FieldMatcher,
|
||||
reducerId: ReducerID[]
|
||||
): DataFrame | undefined {
|
||||
const calculators = fieldReducers.list(reducerId);
|
||||
const reducers = calculators.map(c => c.id);
|
||||
const processed: DataFrame[] = [];
|
||||
|
||||
for (const series of data) {
|
||||
const values: ArrayVector[] = [];
|
||||
const fields: Field[] = [];
|
||||
const byId: KeyValue<ArrayVector> = {};
|
||||
|
||||
values.push(new ArrayVector()); // The name
|
||||
fields.push({
|
||||
name: 'Field',
|
||||
type: FieldType.string,
|
||||
values: values[0],
|
||||
config: {},
|
||||
});
|
||||
|
||||
for (const info of calculators) {
|
||||
const vals = new ArrayVector();
|
||||
byId[info.id] = vals;
|
||||
values.push(vals);
|
||||
|
||||
fields.push({
|
||||
name: info.name,
|
||||
type: FieldType.other, // UNKNOWN until after we call the functions
|
||||
values: values[values.length - 1],
|
||||
config: {},
|
||||
});
|
||||
}
|
||||
|
||||
for (let i = 0; i < series.fields.length; i++) {
|
||||
const field = series.fields[i];
|
||||
|
||||
if (matcher(field, series, data)) {
|
||||
const results = reduceField({
|
||||
field,
|
||||
reducers,
|
||||
});
|
||||
|
||||
// Update the name list
|
||||
const fieldName = getFieldDisplayName(field, series, data);
|
||||
|
||||
values[0].buffer.push(fieldName);
|
||||
|
||||
for (const info of calculators) {
|
||||
const v = results[info.id];
|
||||
byId[info.id].buffer.push(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const f of fields) {
|
||||
const t = guessFieldTypeForField(f);
|
||||
|
||||
if (t) {
|
||||
f.type = t;
|
||||
}
|
||||
}
|
||||
|
||||
processed.push({
|
||||
...series, // Same properties, different fields
|
||||
fields,
|
||||
length: values[0].length,
|
||||
});
|
||||
}
|
||||
|
||||
return mergeResults(processed);
|
||||
}
|
||||
|
||||
/**
|
||||
* @internal only exported for testing
|
||||
*/
|
||||
export function mergeResults(data: DataFrame[]): DataFrame | undefined {
|
||||
if (!data?.length) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const baseFrame = data[0];
|
||||
@ -138,6 +165,50 @@ const mergeResults = (data: DataFrame[]) => {
|
||||
|
||||
baseFrame.name = undefined;
|
||||
baseFrame.length = baseFrame.fields[0].values.length;
|
||||
return baseFrame;
|
||||
}
|
||||
|
||||
return [baseFrame];
|
||||
};
|
||||
/**
|
||||
* @internal -- only exported for testing
|
||||
*/
|
||||
export function reduceFields(data: DataFrame[], matcher: FieldMatcher, reducerId: ReducerID[]): DataFrame[] {
|
||||
const calculators = fieldReducers.list(reducerId);
|
||||
const reducers = calculators.map(c => c.id);
|
||||
const processed: DataFrame[] = [];
|
||||
|
||||
for (const series of data) {
|
||||
const fields: Field[] = [];
|
||||
for (const field of series.fields) {
|
||||
if (matcher(field, series, data)) {
|
||||
const results = reduceField({
|
||||
field,
|
||||
reducers,
|
||||
});
|
||||
for (const reducer of reducers) {
|
||||
const value = results[reducer];
|
||||
const copy = {
|
||||
...field,
|
||||
values: new ArrayVector([value]),
|
||||
};
|
||||
copy.state = undefined;
|
||||
if (reducers.length > 1) {
|
||||
if (!copy.labels) {
|
||||
copy.labels = {};
|
||||
}
|
||||
copy.labels['reducer'] = fieldReducers.get(reducer).name;
|
||||
}
|
||||
fields.push(copy);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (fields.length) {
|
||||
processed.push({
|
||||
...series,
|
||||
fields,
|
||||
length: 1, // always one row
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
@ -110,6 +110,7 @@ export const Components = {
|
||||
},
|
||||
Transforms: {
|
||||
Reduce: {
|
||||
modeLabel: 'Transform mode label',
|
||||
calculationsLabel: 'Transform calculations label',
|
||||
},
|
||||
},
|
||||
|
@ -1,14 +1,15 @@
|
||||
import React from 'react';
|
||||
import { StatsPicker } from '@grafana/ui';
|
||||
import React, { useCallback } from 'react';
|
||||
import { StatsPicker, Select, LegacyForms } from '@grafana/ui';
|
||||
import {
|
||||
DataTransformerID,
|
||||
ReducerID,
|
||||
standardTransformers,
|
||||
TransformerRegistyItem,
|
||||
TransformerUIProps,
|
||||
SelectableValue,
|
||||
} from '@grafana/data';
|
||||
|
||||
import { ReduceTransformerOptions } from '@grafana/data/src/transformations/transformers/reduce';
|
||||
import { ReduceTransformerOptions, ReduceTransformerMode } from '@grafana/data/src/transformations/transformers/reduce';
|
||||
import { selectors } from '@grafana/e2e-selectors';
|
||||
|
||||
// TODO: Minimal implementation, needs some <3
|
||||
@ -16,27 +17,87 @@ export const ReduceTransformerEditor: React.FC<TransformerUIProps<ReduceTransfor
|
||||
options,
|
||||
onChange,
|
||||
}) => {
|
||||
const modes: Array<SelectableValue<ReduceTransformerMode>> = [
|
||||
{
|
||||
label: 'Series to rows',
|
||||
value: ReduceTransformerMode.SeriesToRows,
|
||||
description: 'Create a table with one row for each series value',
|
||||
},
|
||||
{
|
||||
label: 'Reduce fields',
|
||||
value: ReduceTransformerMode.ReduceFields,
|
||||
description: 'Collapse each field into a single value',
|
||||
},
|
||||
];
|
||||
|
||||
const onSelectMode = useCallback(
|
||||
(value: SelectableValue<ReduceTransformerMode>) => {
|
||||
const mode = value.value!;
|
||||
onChange({
|
||||
...options,
|
||||
mode,
|
||||
includeTimeField: mode === ReduceTransformerMode.ReduceFields ? !!options.includeTimeField : false,
|
||||
});
|
||||
},
|
||||
[onChange, options]
|
||||
);
|
||||
|
||||
const onToggleTime = useCallback(() => {
|
||||
onChange({
|
||||
...options,
|
||||
includeTimeField: !options.includeTimeField,
|
||||
});
|
||||
}, [onChange, options]);
|
||||
|
||||
return (
|
||||
<div className="gf-form-inline">
|
||||
<div className="gf-form gf-form--grow">
|
||||
<div className="gf-form-label width-8" aria-label={selectors.components.Transforms.Reduce.calculationsLabel}>
|
||||
Calculations
|
||||
<>
|
||||
<div>
|
||||
<div className="gf-form gf-form--grow">
|
||||
<div className="gf-form-label width-8" aria-label={selectors.components.Transforms.Reduce.modeLabel}>
|
||||
Mode
|
||||
</div>
|
||||
<Select
|
||||
options={modes}
|
||||
value={modes.find(v => v.value === options.mode) || modes[0]}
|
||||
onChange={onSelectMode}
|
||||
menuPlacement="bottom"
|
||||
className="flex-grow-1"
|
||||
/>
|
||||
</div>
|
||||
<StatsPicker
|
||||
className="flex-grow-1"
|
||||
placeholder="Choose Stat"
|
||||
allowMultiple
|
||||
stats={options.reducers || []}
|
||||
onChange={stats => {
|
||||
onChange({
|
||||
...options,
|
||||
reducers: stats as ReducerID[],
|
||||
});
|
||||
}}
|
||||
menuPlacement="bottom"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="gf-form-inline">
|
||||
<div className="gf-form gf-form--grow">
|
||||
<div className="gf-form-label width-8" aria-label={selectors.components.Transforms.Reduce.calculationsLabel}>
|
||||
Calculations
|
||||
</div>
|
||||
<StatsPicker
|
||||
className="flex-grow-1"
|
||||
placeholder="Choose Stat"
|
||||
allowMultiple
|
||||
stats={options.reducers || []}
|
||||
onChange={stats => {
|
||||
onChange({
|
||||
...options,
|
||||
reducers: stats as ReducerID[],
|
||||
});
|
||||
}}
|
||||
menuPlacement="bottom"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{options.mode === ReduceTransformerMode.ReduceFields && (
|
||||
<div className="gf-form-inline">
|
||||
<div className="gf-form">
|
||||
<LegacyForms.Switch
|
||||
label="Include time"
|
||||
labelClass="width-8"
|
||||
checked={!!options.includeTimeField}
|
||||
onChange={onToggleTime}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user