From 093383eb83a8166a1d044ebb41940366e83091f0 Mon Sep 17 00:00:00 2001 From: Marcus Andersson Date: Mon, 15 Jun 2020 15:03:13 +0200 Subject: [PATCH] Transform: added merge transform that will merge multiple series/tables into one table (#25490) * wip: added draft of series to rows. * wip: building dataFrame structure first and then adding data. * wip: added some refactorings of the seriesToRows transformer. * did some refactorings to make the code easier to follow. * added an editor for the transform. * renamed some of the test data. * added docs. * fixed according to feedback. * renamved files. * fixed docs according to feedback. * fixed so we don't keep labels or config values from. * removed unused field. * fixed spelling errors. * fixed docs according to feedback. --- docs/sources/panels/transformations.md | 23 ++ .../src/dataframe/processDataFrame.ts | 26 +- .../src/datetime/moment_wrapper.ts | 1 + .../grafana-data/src/field/fieldComparers.ts | 108 ++++++ .../src/transformations/transformers.ts | 2 + .../src/transformations/transformers/ids.ts | 1 + .../transformers/merge/DataFrameBuilder.ts | 135 +++++++ .../merge/DataFramesStackedByTime.ts | 74 ++++ .../transformers/merge/TimeFieldsByFrame.ts | 39 ++ .../transformers/merge/merge.test.ts | 350 ++++++++++++++++++ .../transformers/merge/merge.ts | 47 +++ .../TransformersUI/MergeTransformerEditor.tsx | 20 + public/app/core/utils/standardTransformers.ts | 2 + 13 files changed, 805 insertions(+), 23 deletions(-) create mode 100644 packages/grafana-data/src/field/fieldComparers.ts create mode 100644 packages/grafana-data/src/transformations/transformers/merge/DataFrameBuilder.ts create mode 100644 packages/grafana-data/src/transformations/transformers/merge/DataFramesStackedByTime.ts create mode 100644 packages/grafana-data/src/transformations/transformers/merge/TimeFieldsByFrame.ts create mode 100644 packages/grafana-data/src/transformations/transformers/merge/merge.test.ts create mode 100644 packages/grafana-data/src/transformations/transformers/merge/merge.ts create mode 100644 public/app/core/components/TransformersUI/MergeTransformerEditor.tsx diff --git a/docs/sources/panels/transformations.md b/docs/sources/panels/transformations.md index cc9d2cdbba5..bc9eb94640a 100644 --- a/docs/sources/panels/transformations.md +++ b/docs/sources/panels/transformations.md @@ -67,6 +67,7 @@ Grafana comes with the following transformations: - [Apply a transformation](#apply-a-transformation) - [Transformation types and options](#transformation-types-and-options) - [Reduce](#reduce) + - [Merge](#merge) - [Filter by name](#filter-by-name) - [Filter data by query](#filter-data-by-query) - [Organize fields](#organize-fields) @@ -93,6 +94,28 @@ After I apply the transformation, there is no time value and each column has bee {{< docs-imagebox img="/img/docs/transformations/reduce-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} +### Merge + +Use this transformation to combine the result from multiple queries into one single result based on the time field. This is helpful when using the table panel visualization. + +In the example below, we are visualizing multiple queries returning table data before applying the transformation. + +{{< docs-imagebox img="/img/docs/transformations/table-data-before-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}} + +Here is the same example after applying the merge transformation. + +{{< docs-imagebox img="/img/docs/transformations/table-data-after-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}} + +If any of the queries return time series data, then a `Metric` column containing the name of the query is added. You can be customized this value by defining `Label` on the source query. + +In the example below, we are visualizing multiple queries returning time series data before applying the transformation. + +{{< docs-imagebox img="/img/docs/transformations/time-series-before-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}} + +Here is the same example after applying the merge transformation. + +{{< docs-imagebox img="/img/docs/transformations/time-series-after-merge-7-1.png" class="docs-image--no-shadow" max-width= "1100px" >}} + ### Filter by name Use this transformation to remove portions of the query results. diff --git a/packages/grafana-data/src/dataframe/processDataFrame.ts b/packages/grafana-data/src/dataframe/processDataFrame.ts index 83efb7e324f..1ee6cfa578c 100644 --- a/packages/grafana-data/src/dataframe/processDataFrame.ts +++ b/packages/grafana-data/src/dataframe/processDataFrame.ts @@ -23,6 +23,7 @@ import { MutableDataFrame } from './MutableDataFrame'; import { SortedVector } from '../vector/SortedVector'; import { ArrayDataFrame } from './ArrayDataFrame'; import { getFieldDisplayName } from '../field/fieldState'; +import { fieldIndexComparer } from '../field/fieldComparers'; function convertTableToDataFrame(table: TableData): DataFrame { const fields = table.columns.map(c => { @@ -391,31 +392,10 @@ export function sortDataFrame(data: DataFrame, sortIndex?: number, reverse = fal for (let i = 0; i < data.length; i++) { index.push(i); } - const values = field.values; - // Numeric Comparison - let compare = (a: number, b: number) => { - const vA = values.get(a); - const vB = values.get(b); - return vA - vB; // works for numbers! - }; + const fieldComparer = fieldIndexComparer(field, reverse); + index.sort(fieldComparer); - // String Comparison - if (field.type === FieldType.string) { - compare = (a: number, b: number) => { - const vA: string = values.get(a); - const vB: string = values.get(b); - return vA.localeCompare(vB); - }; - } - - // Run the sort function - index.sort(compare); - if (reverse) { - index.reverse(); - } - - // Return a copy that maps sorted values return { ...data, fields: data.fields.map(f => { diff --git a/packages/grafana-data/src/datetime/moment_wrapper.ts b/packages/grafana-data/src/datetime/moment_wrapper.ts index 7be53d9e249..b4980c1d8e3 100644 --- a/packages/grafana-data/src/datetime/moment_wrapper.ts +++ b/packages/grafana-data/src/datetime/moment_wrapper.ts @@ -58,6 +58,7 @@ export interface DateTime extends Object { fromNow: (withoutSuffix?: boolean) => string; from: (formaInput: DateTimeInput) => string; isSame: (input?: DateTimeInput, granularity?: DurationUnit) => boolean; + isBefore: (input?: DateTimeInput) => boolean; isValid: () => boolean; local: () => DateTime; locale: (locale: string) => DateTime; diff --git a/packages/grafana-data/src/field/fieldComparers.ts b/packages/grafana-data/src/field/fieldComparers.ts new file mode 100644 index 00000000000..c98c78dbd5b --- /dev/null +++ b/packages/grafana-data/src/field/fieldComparers.ts @@ -0,0 +1,108 @@ +import { Field, FieldType } from '../types/dataFrame'; +import { Vector } from '../types/vector'; +import { dateTime } from '../datetime'; +import isNumber from 'lodash/isNumber'; + +type IndexComparer = (a: number, b: number) => number; + +export const fieldIndexComparer = (field: Field, reverse = false): IndexComparer => { + const values = field.values; + + switch (field.type) { + case FieldType.number: + return numericIndexComparer(values, reverse); + case FieldType.string: + return stringIndexComparer(values, reverse); + case FieldType.boolean: + return booleanIndexComparer(values, reverse); + case FieldType.time: + return timeIndexComparer(values, reverse); + default: + return naturalIndexComparer(reverse); + } +}; + +export const timeComparer = (a: any, b: any): number => { + if (!a || !b) { + return falsyComparer(a, b); + } + + if (isNumber(a) && isNumber(b)) { + return numericComparer(a, b); + } + + if (dateTime(a).isBefore(b)) { + return -1; + } + + if (dateTime(b).isBefore(a)) { + return 1; + } + + return 0; +}; + +export const numericComparer = (a: number, b: number): number => { + return a - b; +}; + +export const stringComparer = (a: string, b: string): number => { + if (!a || !b) { + return falsyComparer(a, b); + } + return a.localeCompare(b); +}; + +export const booleanComparer = (a: boolean, b: boolean): number => { + return falsyComparer(a, b); +}; + +const falsyComparer = (a: any, b: any): number => { + if (!a && b) { + return 1; + } + + if (a && !b) { + return -1; + } + + return 0; +}; + +const timeIndexComparer = (values: Vector, reverse: boolean): IndexComparer => { + return (a: number, b: number): number => { + const vA = values.get(a); + const vB = values.get(b); + return reverse ? timeComparer(vB, vA) : timeComparer(vA, vB); + }; +}; + +const booleanIndexComparer = (values: Vector, reverse: boolean): IndexComparer => { + return (a: number, b: number): number => { + const vA: boolean = values.get(a); + const vB: boolean = values.get(b); + return reverse ? booleanComparer(vB, vA) : booleanComparer(vA, vB); + }; +}; + +const numericIndexComparer = (values: Vector, reverse: boolean): IndexComparer => { + return (a: number, b: number): number => { + const vA: number = values.get(a); + const vB: number = values.get(b); + return reverse ? numericComparer(vB, vA) : numericComparer(vA, vB); + }; +}; + +const stringIndexComparer = (values: Vector, reverse: boolean): IndexComparer => { + return (a: number, b: number): number => { + const vA: string = values.get(a); + const vB: string = values.get(b); + return reverse ? stringComparer(vB, vA) : stringComparer(vA, vB); + }; +}; + +const naturalIndexComparer = (reverse: boolean): IndexComparer => { + return (a: number, b: number): number => { + return reverse ? numericComparer(b, a) : numericComparer(a, b); + }; +}; diff --git a/packages/grafana-data/src/transformations/transformers.ts b/packages/grafana-data/src/transformations/transformers.ts index aad35cfcd68..636fbbe130e 100644 --- a/packages/grafana-data/src/transformations/transformers.ts +++ b/packages/grafana-data/src/transformations/transformers.ts @@ -11,6 +11,7 @@ import { seriesToColumnsTransformer } from './transformers/seriesToColumns'; import { renameFieldsTransformer } from './transformers/rename'; import { labelsToFieldsTransformer } from './transformers/labelsToFields'; import { ensureColumnsTransformer } from './transformers/ensureColumns'; +import { mergeTransformer } from './transformers/merge/merge'; export const standardTransformers = { noopTransformer, @@ -27,4 +28,5 @@ export const standardTransformers = { renameFieldsTransformer, labelsToFieldsTransformer, ensureColumnsTransformer, + mergeTransformer, }; diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index da2c561227b..d12caf2710e 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -8,6 +8,7 @@ export enum DataTransformerID { rename = 'rename', calculateField = 'calculateField', seriesToColumns = 'seriesToColumns', + merge = 'merge', labelsToFields = 'labelsToFields', filterFields = 'filterFields', filterFieldsByName = 'filterFieldsByName', diff --git a/packages/grafana-data/src/transformations/transformers/merge/DataFrameBuilder.ts b/packages/grafana-data/src/transformations/transformers/merge/DataFrameBuilder.ts new file mode 100644 index 00000000000..9f36435b1e1 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/merge/DataFrameBuilder.ts @@ -0,0 +1,135 @@ +import { MutableDataFrame } from '../../../dataframe'; +import { + DataFrame, + FieldType, + Field, + TIME_SERIES_TIME_FIELD_NAME, + TIME_SERIES_VALUE_FIELD_NAME, +} from '../../../types/dataFrame'; +import { ArrayVector } from '../../../vector'; +import { omit } from 'lodash'; +import { getFrameDisplayName } from '../../../field'; + +interface DataFrameBuilderResult { + dataFrame: MutableDataFrame; + valueMapper: ValueMapper; +} + +type ValueMapper = (frame: DataFrame, valueIndex: number, timeIndex: number) => Record; + +const TIME_SERIES_METRIC_FIELD_NAME = 'Metric'; + +export class DataFrameBuilder { + private isOnlyTimeSeries: boolean; + private displayMetricField: boolean; + private valueFields: Record; + private timeField: Field | null; + + constructor() { + this.isOnlyTimeSeries = true; + this.displayMetricField = false; + this.valueFields = {}; + this.timeField = null; + } + + addFields(frame: DataFrame, timeIndex: number): void { + if (frame.fields.length > 2) { + this.isOnlyTimeSeries = false; + } + + if (frame.fields.length === 2) { + this.displayMetricField = true; + } + + for (let index = 0; index < frame.fields.length; index++) { + const field = frame.fields[index]; + + if (index === timeIndex) { + if (!this.timeField) { + this.timeField = this.copyStructure(field, TIME_SERIES_TIME_FIELD_NAME); + } + continue; + } + + if (!this.valueFields[field.name]) { + this.valueFields[field.name] = this.copyStructure(field, field.name); + } + } + } + + build(): DataFrameBuilderResult { + return { + dataFrame: this.createDataFrame(), + valueMapper: this.createValueMapper(), + }; + } + + private createValueMapper(): ValueMapper { + return (frame: DataFrame, valueIndex: number, timeIndex: number) => { + return frame.fields.reduce((values: Record, field, index) => { + const value = field.values.get(valueIndex); + + if (index === timeIndex) { + values[TIME_SERIES_TIME_FIELD_NAME] = value; + + if (this.displayMetricField) { + values[TIME_SERIES_METRIC_FIELD_NAME] = getFrameDisplayName(frame); + } + return values; + } + + if (this.isOnlyTimeSeries) { + values[TIME_SERIES_VALUE_FIELD_NAME] = value; + return values; + } + + values[field.name] = value; + return values; + }, {}); + }; + } + + private createDataFrame(): MutableDataFrame { + const dataFrame = new MutableDataFrame(); + + if (this.timeField) { + dataFrame.addField(this.timeField); + + if (this.displayMetricField) { + dataFrame.addField({ + name: TIME_SERIES_METRIC_FIELD_NAME, + type: FieldType.string, + }); + } + } + + const valueFields = Object.values(this.valueFields); + + if (this.isOnlyTimeSeries) { + if (valueFields.length > 0) { + dataFrame.addField({ + ...valueFields[0], + name: TIME_SERIES_VALUE_FIELD_NAME, + }); + } + return dataFrame; + } + + for (const field of valueFields) { + dataFrame.addField(field); + } + + return dataFrame; + } + + private copyStructure(field: Field, name: string): Field { + return { + ...omit(field, ['values', 'name', 'state', 'labels', 'config']), + name, + values: new ArrayVector(), + config: { + ...omit(field.config, 'displayName'), + }, + }; + } +} diff --git a/packages/grafana-data/src/transformations/transformers/merge/DataFramesStackedByTime.ts b/packages/grafana-data/src/transformations/transformers/merge/DataFramesStackedByTime.ts new file mode 100644 index 00000000000..3d19acc605a --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/merge/DataFramesStackedByTime.ts @@ -0,0 +1,74 @@ +import { DataFrame } from '../../../types/dataFrame'; +import { timeComparer } from '../../../field/fieldComparers'; +import { sortDataFrame } from '../../../dataframe'; +import { TimeFieldsByFrame } from './TimeFieldsByFrame'; + +interface DataFrameStackValue { + valueIndex: number; + timeIndex: number; + frame: DataFrame; +} +export class DataFramesStackedByTime { + private valuesPointerByFrame: Record; + private dataFrames: DataFrame[]; + private isSorted: boolean; + + constructor(private timeFields: TimeFieldsByFrame) { + this.valuesPointerByFrame = {}; + this.dataFrames = []; + this.isSorted = false; + } + + push(frame: DataFrame): number { + const index = this.dataFrames.length; + this.valuesPointerByFrame[index] = 0; + this.dataFrames.push(frame); + return index; + } + + pop(): DataFrameStackValue { + if (!this.isSorted) { + this.sortByTime(); + this.isSorted = true; + } + + const frameIndex = this.dataFrames.reduce((champion, frame, index) => { + const championTime = this.peekTimeValueForFrame(champion); + const contenderTime = this.peekTimeValueForFrame(index); + return timeComparer(contenderTime, championTime) >= 0 ? champion : index; + }, 0); + + const previousPointer = this.movePointerForward(frameIndex); + + return { + frame: this.dataFrames[frameIndex], + valueIndex: previousPointer, + timeIndex: this.timeFields.getFieldIndex(frameIndex), + }; + } + + getLength(): number { + const frames = Object.values(this.dataFrames); + return frames.reduce((length: number, frame) => (length += frame.length), 0); + } + + private peekTimeValueForFrame(frameIndex: number): any { + const timeField = this.timeFields.getField(frameIndex); + const valuePointer = this.valuesPointerByFrame[frameIndex]; + return timeField.values.get(valuePointer); + } + + private movePointerForward(frameIndex: number): number { + const currentPointer = this.valuesPointerByFrame[frameIndex]; + this.valuesPointerByFrame[frameIndex] = currentPointer + 1; + + return currentPointer; + } + + private sortByTime() { + this.dataFrames = this.dataFrames.map((frame, index) => { + const timeFieldIndex = this.timeFields.getFieldIndex(index); + return sortDataFrame(frame, timeFieldIndex); + }); + } +} diff --git a/packages/grafana-data/src/transformations/transformers/merge/TimeFieldsByFrame.ts b/packages/grafana-data/src/transformations/transformers/merge/TimeFieldsByFrame.ts new file mode 100644 index 00000000000..c1f44d6d516 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/merge/TimeFieldsByFrame.ts @@ -0,0 +1,39 @@ +import { isNumber } from 'lodash'; +import { Field, DataFrame } from '../../../types/dataFrame'; +import { getTimeField } from '../../../dataframe'; + +export class TimeFieldsByFrame { + private timeIndexByFrameIndex: Record; + private timeFieldByFrameIndex: Record; + + constructor() { + this.timeIndexByFrameIndex = {}; + this.timeFieldByFrameIndex = {}; + } + + add(frameIndex: number, frame: DataFrame): void { + const fieldDescription = getTimeField(frame); + const timeIndex = fieldDescription?.timeIndex; + const timeField = fieldDescription?.timeField; + + if (isNumber(timeIndex)) { + this.timeIndexByFrameIndex[frameIndex] = timeIndex; + } + + if (timeField) { + this.timeFieldByFrameIndex[frameIndex] = timeField; + } + } + + getField(frameIndex: number): Field { + return this.timeFieldByFrameIndex[frameIndex]; + } + + getFieldIndex(frameIndex: number): number { + return this.timeIndexByFrameIndex[frameIndex]; + } + + getLength() { + return Object.keys(this.timeIndexByFrameIndex).length; + } +} diff --git a/packages/grafana-data/src/transformations/transformers/merge/merge.test.ts b/packages/grafana-data/src/transformations/transformers/merge/merge.test.ts new file mode 100644 index 00000000000..e27ac109d44 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/merge/merge.test.ts @@ -0,0 +1,350 @@ +import { mockTransformationsRegistry } from '../../../utils/tests/mockTransformationsRegistry'; +import { DataTransformerConfig, Field, FieldType } from '../../../types'; +import { DataTransformerID } from '../ids'; +import { toDataFrame } from '../../../dataframe'; +import { transformDataFrame } from '../../transformDataFrame'; +import { ArrayVector } from '../../../vector'; +import { mergeTransformer, MergeTransformerOptions } from './merge'; + +describe('Merge multipe to single', () => { + beforeAll(() => { + mockTransformationsRegistry([mergeTransformer]); + }); + + it('combine two series into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.merge, + options: {}, + }; + + const seriesA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [1000] }, + { name: 'Temp', type: FieldType.number, values: [1] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [2000] }, + { name: 'Temp', type: FieldType.number, values: [-1] }, + ], + }); + + const result = transformDataFrame([cfg], [seriesA, seriesB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [1000, 2000]), + createField('Metric', FieldType.string, ['A', 'B']), + createField('Value', FieldType.number, [1, -1]), + ]; + + expect(result[0].fields).toMatchObject(expected); + }); + + it('combine two series with multiple values into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.merge, + options: {}, + }; + + const seriesA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 150, 200] }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 125, 126] }, + { name: 'Temp', type: FieldType.number, values: [-1, 2, 3] }, + ], + }); + + const result = transformDataFrame([cfg], [seriesA, seriesB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]), + createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']), + createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5]), + ]; + + expect(result[0].fields).toMatchObject(expected); + }); + + it('combine three series into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.merge, + options: {}, + }; + + const seriesA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [1000] }, + { name: 'Temp', type: FieldType.number, values: [1] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [2000] }, + { name: 'Temp', type: FieldType.number, values: [-1] }, + ], + }); + + const seriesC = toDataFrame({ + name: 'C', + fields: [ + { name: 'Time', type: FieldType.time, values: [500] }, + { name: 'Temp', type: FieldType.number, values: [2] }, + ], + }); + + const result = transformDataFrame([cfg], [seriesA, seriesB, seriesC]); + const expected: Field[] = [ + createField('Time', FieldType.time, [500, 1000, 2000]), + createField('Metric', FieldType.string, ['C', 'A', 'B']), + createField('Value', FieldType.number, [2, 1, -1]), + ]; + + expect(result[0].fields).toMatchObject(expected); + }); + + it('combine one serie and two tables into one table', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.merge, + options: {}, + }; + + const tableA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [1000] }, + { name: 'Temp', type: FieldType.number, values: [1] }, + { name: 'Humidity', type: FieldType.number, values: [10] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [1000] }, + { name: 'Temp', type: FieldType.number, values: [-1] }, + ], + }); + + const tableB = toDataFrame({ + name: 'C', + fields: [ + { name: 'Time', type: FieldType.time, values: [500] }, + { name: 'Temp', type: FieldType.number, values: [2] }, + { name: 'Humidity', type: FieldType.number, values: [5] }, + ], + }); + + const result = transformDataFrame([cfg], [tableA, seriesB, tableB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [500, 1000, 1000]), + createField('Metric', FieldType.string, ['C', 'A', 'B']), + createField('Temp', FieldType.number, [2, 1, -1]), + createField('Humidity', FieldType.number, [5, 10, null]), + ]; + + expect(result[0].fields).toMatchObject(expected); + }); + + it('combine one serie and two tables with ISO dates into one table', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.merge, + options: {}, + }; + + const tableA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: ['2019-10-01T11:10:23Z'] }, + { name: 'Temp', type: FieldType.number, values: [1] }, + { name: 'Humidity', type: FieldType.number, values: [10] }, + ], + }); + + const seriesB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: ['2019-09-01T11:10:23Z'] }, + { name: 'Temp', type: FieldType.number, values: [-1] }, + ], + }); + + const tableC = toDataFrame({ + name: 'C', + fields: [ + { name: 'Time', type: FieldType.time, values: ['2019-11-01T11:10:23Z'] }, + { name: 'Temp', type: FieldType.number, values: [2] }, + { name: 'Humidity', type: FieldType.number, values: [5] }, + ], + }); + + const result = transformDataFrame([cfg], [tableA, seriesB, tableC]); + const expected: Field[] = [ + createField('Time', FieldType.time, ['2019-09-01T11:10:23Z', '2019-10-01T11:10:23Z', '2019-11-01T11:10:23Z']), + createField('Metric', FieldType.string, ['B', 'A', 'C']), + createField('Temp', FieldType.number, [-1, 1, 2]), + createField('Humidity', FieldType.number, [null, 10, 5]), + ]; + + expect(result[0].fields).toMatchObject(expected); + }); + + it('combine three tables with multiple values into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.merge, + options: {}, + }; + + const tableA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 150, 200] }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5] }, + { name: 'Humidity', type: FieldType.number, values: [10, 14, 55] }, + ], + }); + + const tableB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 125, 126] }, + { name: 'Temp', type: FieldType.number, values: [-1, 2, 3] }, + { name: 'Enabled', type: FieldType.boolean, values: [true, false, true] }, + ], + }); + + const tableC = toDataFrame({ + name: 'C', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 124, 149] }, + { name: 'Humidity', type: FieldType.number, values: [22, 25, 30] }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5] }, + ], + }); + + const result = transformDataFrame([cfg], [tableA, tableB, tableC]); + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 100, 100, 124, 125, 126, 149, 150, 200]), + createField('Temp', FieldType.number, [1, -1, 1, 4, 2, 3, 5, 4, 5]), + createField('Humidity', FieldType.number, [10, null, 22, 25, null, null, 30, 14, 55]), + createField('Enabled', FieldType.boolean, [null, true, null, null, false, true, null, null, null]), + ]; + + expect(result[0].fields).toMatchObject(expected); + }); + + it('combine two time series, where first serie fields has displayName, into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.merge, + options: {}, + }; + + const serieA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 150, 200], config: { displayName: 'Random time' } }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5], config: { displayName: 'Temp' } }, + ], + }); + + const serieB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 125, 126] }, + { name: 'Temp', type: FieldType.number, values: [-1, 2, 3] }, + ], + }); + + const result = transformDataFrame([cfg], [serieA, serieB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]), + createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']), + createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5]), + ]; + + expect(result[0].fields[2].config).toEqual({}); + expect(result[0].fields).toMatchObject(expected); + }); + + it('combine two time series, where first serie fields has units, into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.merge, + options: {}, + }; + + const serieA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 150, 200] }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5], config: { units: 'celsius' } }, + ], + }); + + const serieB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 125, 126] }, + { name: 'Temp', type: FieldType.number, values: [-1, 2, 3] }, + ], + }); + + const result = transformDataFrame([cfg], [serieA, serieB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]), + createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']), + createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5], { units: 'celsius' }), + ]; + + expect(result[0].fields[2].config).toEqual({ units: 'celsius' }); + expect(result[0].fields).toMatchObject(expected); + }); + + it('combine two time series, where second serie fields has units, into one', () => { + const cfg: DataTransformerConfig = { + id: DataTransformerID.merge, + options: {}, + }; + + const serieA = toDataFrame({ + name: 'A', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 150, 200] }, + { name: 'Temp', type: FieldType.number, values: [1, 4, 5] }, + ], + }); + + const serieB = toDataFrame({ + name: 'B', + fields: [ + { name: 'Time', type: FieldType.time, values: [100, 125, 126] }, + { name: 'Temp', type: FieldType.number, values: [-1, 2, 3], config: { units: 'celsius' } }, + ], + }); + + const result = transformDataFrame([cfg], [serieA, serieB]); + const expected: Field[] = [ + createField('Time', FieldType.time, [100, 100, 125, 126, 150, 200]), + createField('Metric', FieldType.string, ['A', 'B', 'B', 'B', 'A', 'A']), + createField('Value', FieldType.number, [1, -1, 2, 3, 4, 5]), + ]; + + expect(result[0].fields[2].config).toEqual({}); + expect(result[0].fields).toMatchObject(expected); + }); +}); + +const createField = (name: string, type: FieldType, values: any[], config = {}): Field => { + return { name, type, values: new ArrayVector(values), config, labels: undefined }; +}; diff --git a/packages/grafana-data/src/transformations/transformers/merge/merge.ts b/packages/grafana-data/src/transformations/transformers/merge/merge.ts new file mode 100644 index 00000000000..15d7b70f99a --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/merge/merge.ts @@ -0,0 +1,47 @@ +import { DataTransformerID } from '../ids'; +import { DataTransformerInfo } from '../../../types/transformations'; +import { DataFrame } from '../../../types/dataFrame'; +import { DataFrameBuilder } from './DataFrameBuilder'; +import { TimeFieldsByFrame } from './TimeFieldsByFrame'; +import { DataFramesStackedByTime } from './DataFramesStackedByTime'; + +export interface MergeTransformerOptions {} + +export const mergeTransformer: DataTransformerInfo = { + id: DataTransformerID.merge, + name: 'Merge series/tables', + description: 'Merges multiple series/tables by time into a single serie/table', + defaultOptions: {}, + transformer: (options: MergeTransformerOptions) => { + return (data: DataFrame[]) => { + if (!Array.isArray(data) || data.length <= 1) { + return data; + } + + const timeFields = new TimeFieldsByFrame(); + const framesStack = new DataFramesStackedByTime(timeFields); + const dataFrameBuilder = new DataFrameBuilder(); + + for (const frame of data) { + const frameIndex = framesStack.push(frame); + timeFields.add(frameIndex, frame); + + const timeIndex = timeFields.getFieldIndex(frameIndex); + dataFrameBuilder.addFields(frame, timeIndex); + } + + if (data.length !== timeFields.getLength()) { + return data; + } + + const { dataFrame, valueMapper } = dataFrameBuilder.build(); + + for (let index = 0; index < framesStack.getLength(); index++) { + const { frame, valueIndex, timeIndex } = framesStack.pop(); + dataFrame.add(valueMapper(frame, valueIndex, timeIndex)); + } + + return [dataFrame]; + }; + }, +}; diff --git a/public/app/core/components/TransformersUI/MergeTransformerEditor.tsx b/public/app/core/components/TransformersUI/MergeTransformerEditor.tsx new file mode 100644 index 00000000000..dfa4b723e5d --- /dev/null +++ b/public/app/core/components/TransformersUI/MergeTransformerEditor.tsx @@ -0,0 +1,20 @@ +import React from 'react'; +import { DataTransformerID, standardTransformers, TransformerRegistyItem, TransformerUIProps } from '@grafana/data'; +import { MergeTransformerOptions } from '@grafana/data/src/transformations/transformers/merge/merge'; + +export const MergeTransformerEditor: React.FC> = ({ + input, + options, + onChange, +}) => { + return null; +}; + +export const mergeTransformerRegistryItem: TransformerRegistyItem = { + id: DataTransformerID.merge, + editor: MergeTransformerEditor, + transformation: standardTransformers.mergeTransformer, + name: 'Merge on time', + description: `Merge series/tables by time and return a single table with values as rows. + Useful for showing multiple time series, tables or a combination of both visualized in a table.`, +}; diff --git a/public/app/core/utils/standardTransformers.ts b/public/app/core/utils/standardTransformers.ts index 0c682873458..f7437d40624 100644 --- a/public/app/core/utils/standardTransformers.ts +++ b/public/app/core/utils/standardTransformers.ts @@ -6,6 +6,7 @@ import { organizeFieldsTransformRegistryItem } from '../components/TransformersU import { seriesToFieldsTransformerRegistryItem } from '../components/TransformersUI/SeriesToFieldsTransformerEditor'; import { calculateFieldTransformRegistryItem } from '../components/TransformersUI/CalculateFieldTransformerEditor'; import { labelsToFieldsTransformerRegistryItem } from '../components/TransformersUI/LabelsToFieldsTransformerEditor'; +import { mergeTransformerRegistryItem } from '../components/TransformersUI/MergeTransformerEditor'; export const getStandardTransformers = (): Array> => { return [ @@ -16,5 +17,6 @@ export const getStandardTransformers = (): Array> => seriesToFieldsTransformerRegistryItem, calculateFieldTransformRegistryItem, labelsToFieldsTransformerRegistryItem, + mergeTransformerRegistryItem, ]; };