From 5307cfeabd12dd2bc6ab53a507c72a9962cb6737 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hugo=20H=C3=A4ggmark?= Date: Mon, 30 Mar 2020 06:24:54 +0200 Subject: [PATCH] Transformers: adds series to column transformer (#23012) * Refactor: adds first naive implemenation of join by field name * Chore: changes after PR comments * Refactor: fixes labels and adds support for multiple columns --- .../matchers/fieldTypeMatcher.ts | 12 +- .../src/transformations/transformers.ts | 10 +- .../src/transformations/transformers/ids.ts | 3 +- .../transformers/seriesToColumns.test.ts | 177 ++++++++++++++++++ .../transformers/seriesToColumns.ts | 123 ++++++++++++ 5 files changed, 314 insertions(+), 11 deletions(-) create mode 100644 packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts create mode 100644 packages/grafana-data/src/transformations/transformers/seriesToColumns.ts diff --git a/packages/grafana-data/src/transformations/matchers/fieldTypeMatcher.ts b/packages/grafana-data/src/transformations/matchers/fieldTypeMatcher.ts index 38dc9c5ef0a..4a9d2dffe43 100644 --- a/packages/grafana-data/src/transformations/matchers/fieldTypeMatcher.ts +++ b/packages/grafana-data/src/transformations/matchers/fieldTypeMatcher.ts @@ -3,7 +3,7 @@ import { FieldMatcherID } from './ids'; import { FieldMatcherInfo } from '../../types/transformations'; // General Field matcher -const fieldTypeMacher: FieldMatcherInfo = { +const fieldTypeMatcher: FieldMatcherInfo = { id: FieldMatcherID.byType, name: 'Field Type', description: 'match based on the field type', @@ -22,13 +22,13 @@ const fieldTypeMacher: FieldMatcherInfo = { // Numeric Field matcher // This gets its own entry so it shows up in the dropdown -const numericMacher: FieldMatcherInfo = { +const numericMatcher: FieldMatcherInfo = { id: FieldMatcherID.numeric, name: 'Numeric Fields', description: 'Fields with type number', get: () => { - return fieldTypeMacher.get(FieldType.number); + return fieldTypeMatcher.get(FieldType.number); }, getOptionsDisplayText: () => { @@ -37,13 +37,13 @@ const numericMacher: FieldMatcherInfo = { }; // Time Field matcher -const timeMacher: FieldMatcherInfo = { +const timeMatcher: FieldMatcherInfo = { id: FieldMatcherID.time, name: 'Time Fields', description: 'Fields with type time', get: () => { - return fieldTypeMacher.get(FieldType.time); + return fieldTypeMatcher.get(FieldType.time); }, getOptionsDisplayText: () => { @@ -55,5 +55,5 @@ const timeMacher: FieldMatcherInfo = { * Registry Initalization */ export function getFieldTypeMatchers(): FieldMatcherInfo[] { - return [fieldTypeMacher, numericMacher, timeMacher]; + return [fieldTypeMatcher, numericMatcher, timeMatcher]; } diff --git a/packages/grafana-data/src/transformations/transformers.ts b/packages/grafana-data/src/transformations/transformers.ts index dc2e1489ae1..09ec993e043 100644 --- a/packages/grafana-data/src/transformations/transformers.ts +++ b/packages/grafana-data/src/transformations/transformers.ts @@ -1,14 +1,15 @@ import { DataFrame } from '../types/dataFrame'; import { Registry } from '../utils/Registry'; -// Initalize the Registry - -import { appendTransformer, AppendOptions } from './transformers/append'; +import { AppendOptions, appendTransformer } from './transformers/append'; import { reduceTransformer, ReduceTransformerOptions } from './transformers/reduce'; import { filterFieldsTransformer, filterFramesTransformer } from './transformers/filter'; import { filterFieldsByNameTransformer, FilterFieldsByNameTransformerOptions } from './transformers/filterByName'; import { noopTransformer } from './transformers/noop'; -import { DataTransformerInfo, DataTransformerConfig } from '../types/transformations'; +import { DataTransformerConfig, DataTransformerInfo } from '../types/transformations'; import { filterFramesByRefIdTransformer } from './transformers/filterByRefId'; +import { seriesToColumnsTransformer } from './transformers/seriesToColumns'; + +// Initalize the Registry /** * Apply configured transformations to the input data @@ -68,6 +69,7 @@ export const transformersRegistry = new TransformerRegistry(() => [ filterFramesByRefIdTransformer, appendTransformer, reduceTransformer, + seriesToColumnsTransformer, ]); export { ReduceTransformerOptions, FilterFieldsByNameTransformerOptions }; diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index 79bcb63ab81..aeeefca06bc 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -1,9 +1,10 @@ export enum DataTransformerID { - // join = 'join', // Pick a field and merge all series based on that field + // join = 'join', // Pick a field and merge all series based on that field append = 'append', // Merge all series together // rotate = 'rotate', // Columns to rows reduce = 'reduce', // Run calculations on fields + seriesToColumns = 'seriesToColumns', // former table transform timeseries_to_columns filterFields = 'filterFields', // Pick some fields (keep all frames) filterFieldsByName = 'filterFieldsByName', // Pick fields with name matching regex (keep all frames) filterFrames = 'filterFrames', // Pick some frames (keep all fields) diff --git a/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts b/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts new file mode 100644 index 00000000000..36f49b0487d --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts @@ -0,0 +1,177 @@ +import { + ArrayVector, + DataTransformerConfig, + DataTransformerID, + FieldType, + toDataFrame, + transformDataFrame, +} from '@grafana/data'; +import { SeriesToColumnsOptions } from './seriesToColumns'; + +describe('SeriesToColumns Transformer', () => { + const everySecondSeries = toDataFrame({ + name: 'even', + fields: [ + { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] }, + { name: 'temperature', type: FieldType.number, values: [10.3, 10.4, 10.5, 10.6] }, + { name: 'humidity', type: FieldType.number, values: [10000.3, 10000.4, 10000.5, 10000.6] }, + ], + }); + + const everyOtherSecondSeries = toDataFrame({ + name: 'odd', + fields: [ + { name: 'time', type: FieldType.time, values: [1000, 3000, 5000, 7000] }, + { name: 'temperature', type: FieldType.number, values: [11.1, 11.3, 11.5, 11.7] }, + { name: 'humidity', type: FieldType.number, values: [11000.1, 11000.3, 11000.5, 11000.7] }, + ], + }); + + it('joins by time field', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + }, + }; + + const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0]; + expect(filtered.fields).toEqual([ + { + name: 'time', + type: FieldType.time, + values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]), + config: {}, + labels: { origin: 'even,odd' }, + }, + { + name: 'temperature {even}', + type: FieldType.number, + values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]), + config: {}, + labels: { origin: 'even' }, + }, + { + name: 'humidity {even}', + type: FieldType.number, + values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]), + config: {}, + labels: { origin: 'even' }, + }, + { + name: 'temperature {odd}', + type: FieldType.number, + values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]), + config: {}, + labels: { origin: 'odd' }, + }, + { + name: 'humidity {odd}', + type: FieldType.number, + values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]), + config: {}, + labels: { origin: 'odd' }, + }, + ]); + }); + + it('joins by temperature field', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'temperature', + }, + }; + + const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0]; + expect(filtered.fields).toEqual([ + { + name: 'temperature', + type: FieldType.number, + values: new ArrayVector([10.3, 10.4, 10.5, 10.6, 11.1, 11.3, 11.5, 11.7]), + config: {}, + labels: { origin: 'even,odd' }, + }, + { + name: 'time {even}', + type: FieldType.time, + values: new ArrayVector([3000, 4000, 5000, 6000, null, null, null, null]), + config: {}, + labels: { origin: 'even' }, + }, + { + name: 'humidity {even}', + type: FieldType.number, + values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6, null, null, null, null]), + config: {}, + labels: { origin: 'even' }, + }, + { + name: 'time {odd}', + type: FieldType.time, + values: new ArrayVector([null, null, null, null, 1000, 3000, 5000, 7000]), + config: {}, + labels: { origin: 'odd' }, + }, + { + name: 'humidity {odd}', + type: FieldType.number, + values: new ArrayVector([null, null, null, null, 11000.1, 11000.3, 11000.5, 11000.7]), + config: {}, + labels: { origin: 'odd' }, + }, + ]); + }); + + it('joins by time field in reverse order', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.seriesToColumns, + options: { + byField: 'time', + }, + }; + + everySecondSeries.fields[0].values = new ArrayVector(everySecondSeries.fields[0].values.toArray().reverse()); + everySecondSeries.fields[1].values = new ArrayVector(everySecondSeries.fields[1].values.toArray().reverse()); + everySecondSeries.fields[2].values = new ArrayVector(everySecondSeries.fields[2].values.toArray().reverse()); + + const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0]; + expect(filtered.fields).toEqual([ + { + name: 'time', + type: FieldType.time, + values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]), + config: {}, + labels: { origin: 'even,odd' }, + }, + { + name: 'temperature {even}', + type: FieldType.number, + values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]), + config: {}, + labels: { origin: 'even' }, + }, + { + name: 'humidity {even}', + type: FieldType.number, + values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]), + config: {}, + labels: { origin: 'even' }, + }, + { + name: 'temperature {odd}', + type: FieldType.number, + values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]), + config: {}, + labels: { origin: 'odd' }, + }, + { + name: 'humidity {odd}', + type: FieldType.number, + values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]), + config: {}, + labels: { origin: 'odd' }, + }, + ]); + }); +}); diff --git a/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts b/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts new file mode 100644 index 00000000000..d8bad654523 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/seriesToColumns.ts @@ -0,0 +1,123 @@ +import { DataFrame, DataTransformerInfo } from '../../types'; +import { DataTransformerID } from './ids'; +import { MutableDataFrame } from '../../dataframe'; +import { filterFieldsByNameTransformer } from './filterByName'; +import { ArrayVector } from '../../vector'; + +export interface SeriesToColumnsOptions { + byField: string; +} + +export const seriesToColumnsTransformer: DataTransformerInfo = { + id: DataTransformerID.seriesToColumns, + name: 'Series as Columns', + description: 'Groups series by field and returns values as columns', + defaultOptions: {}, + transformer: options => (data: DataFrame[]) => { + const regex = `/^(${options.byField})$/`; + // not sure if I should use filterFieldsByNameTransformer to get the key field + const keyDataFrames = filterFieldsByNameTransformer.transformer({ include: regex })(data); + if (!keyDataFrames.length) { + // for now we only parse data frames with 2 fields + return data; + } + + // not sure if I should use filterFieldsByNameTransformer to get the other fields + const otherDataFrames = filterFieldsByNameTransformer.transformer({ exclude: regex })(data); + if (!otherDataFrames.length) { + // for now we only parse data frames with 2 fields + return data; + } + + const processed = new MutableDataFrame(); + const origins: string[] = []; + for (let frameIndex = 0; frameIndex < keyDataFrames.length; frameIndex++) { + const frame = keyDataFrames[frameIndex]; + const origin = getOrigin(frame, frameIndex); + origins.push(origin); + } + + processed.addField({ + ...keyDataFrames[0].fields[0], + values: new ArrayVector([]), + labels: { origin: origins.join(',') }, + }); + + for (let frameIndex = 0; frameIndex < otherDataFrames.length; frameIndex++) { + const frame = otherDataFrames[frameIndex]; + for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) { + const field = frame.fields[fieldIndex]; + const origin = getOrigin(frame, frameIndex); + const name = getColumnName(otherDataFrames, frameIndex, fieldIndex, false); + if (processed.fields.find(field => field.name === name)) { + continue; + } + processed.addField({ ...field, name, values: new ArrayVector([]), labels: { origin } }); + } + } + + const byKeyField: { [key: string]: { [key: string]: any } } = {}; + // this loop creates a dictionary object that groups the key fields values + /* + { + "key field first value as string" : { + "key field name": key field first value, + "other series name": other series value + "other series n name": other series n value + }, + "key field n value as string" : { + "key field name": key field n value, + "other series name": other series value + "other series n name": other series n value + } + } + */ + for (let seriesIndex = 0; seriesIndex < keyDataFrames.length; seriesIndex++) { + const keyDataFrame = keyDataFrames[seriesIndex]; + const keyField = keyDataFrame.fields[0]; + const keyColumnName = getColumnName(keyDataFrames, seriesIndex, 0, true); + const keyValues = keyField.values; + for (let valueIndex = 0; valueIndex < keyValues.length; valueIndex++) { + const keyValue = keyValues.get(valueIndex); + const keyValueAsString = keyValue.toString(); + if (!byKeyField[keyValueAsString]) { + byKeyField[keyValueAsString] = { [keyColumnName]: keyValue }; + } + const otherDataFrame = otherDataFrames[seriesIndex]; + for (let otherIndex = 0; otherIndex < otherDataFrame.fields.length; otherIndex++) { + const otherColumnName = getColumnName(otherDataFrames, seriesIndex, otherIndex, false); + const otherField = otherDataFrame.fields[otherIndex]; + const otherValue = otherField.values.get(valueIndex); + if (!byKeyField[keyValueAsString][otherColumnName]) { + byKeyField[keyValueAsString] = { ...byKeyField[keyValueAsString], [otherColumnName]: otherValue }; + } + } + } + } + + const keyValueStrings = Object.keys(byKeyField); + for (let rowIndex = 0; rowIndex < keyValueStrings.length; rowIndex++) { + const keyValueAsString = keyValueStrings[rowIndex]; + for (let fieldIndex = 0; fieldIndex < processed.fields.length; fieldIndex++) { + const field = processed.fields[fieldIndex]; + const value = byKeyField[keyValueAsString][field.name] ?? null; + field.values.add(value); + } + } + + return [processed]; + }, +}; + +const getColumnName = (frames: DataFrame[], frameIndex: number, fieldIndex: number, isKeyField = false) => { + const frame = frames[frameIndex]; + const frameName = frame.name || `${frameIndex}`; + const fieldName = frame.fields[fieldIndex].name; + const seriesName = isKeyField ? fieldName : `${fieldName} {${frameName}}`; + + return seriesName; +}; + +const getOrigin = (frame: DataFrame, index: number) => { + return frame.name || `${index}`; +};