Transformers: adds series to column transformer (#23012)

* Refactor: adds first naive implemenation of join by field name

* Chore: changes after PR comments

* Refactor: fixes labels and adds support for multiple columns

packages/grafana-data/src/transformations/matchers/fieldTypeMatcher.ts

@@ -3,7 +3,7 @@ import { FieldMatcherID } from './ids';
 import { FieldMatcherInfo } from '../../types/transformations';
 
 // General Field matcher
-const fieldTypeMacher: FieldMatcherInfo<FieldType> = {
+const fieldTypeMatcher: FieldMatcherInfo<FieldType> = {
   id: FieldMatcherID.byType,
   name: 'Field Type',
   description: 'match based on the field type',
@@ -22,13 +22,13 @@ const fieldTypeMacher: FieldMatcherInfo<FieldType> = {
 
 // Numeric Field matcher
 // This gets its own entry so it shows up in the dropdown
-const numericMacher: FieldMatcherInfo = {
+const numericMatcher: FieldMatcherInfo = {
   id: FieldMatcherID.numeric,
   name: 'Numeric Fields',
   description: 'Fields with type number',
 
   get: () => {
-    return fieldTypeMacher.get(FieldType.number);
+    return fieldTypeMatcher.get(FieldType.number);
   },
 
   getOptionsDisplayText: () => {
@@ -37,13 +37,13 @@ const numericMacher: FieldMatcherInfo = {
 };
 
 // Time Field matcher
-const timeMacher: FieldMatcherInfo = {
+const timeMatcher: FieldMatcherInfo = {
   id: FieldMatcherID.time,
   name: 'Time Fields',
   description: 'Fields with type time',
 
   get: () => {
-    return fieldTypeMacher.get(FieldType.time);
+    return fieldTypeMatcher.get(FieldType.time);
   },
 
   getOptionsDisplayText: () => {
@@ -55,5 +55,5 @@ const timeMacher: FieldMatcherInfo = {
  * Registry Initalization
  */
 export function getFieldTypeMatchers(): FieldMatcherInfo[] {
-  return [fieldTypeMacher, numericMacher, timeMacher];
+  return [fieldTypeMatcher, numericMatcher, timeMatcher];
 }

packages/grafana-data/src/transformations/transformers.ts

@@ -1,14 +1,15 @@
 import { DataFrame } from '../types/dataFrame';
 import { Registry } from '../utils/Registry';
-// Initalize the Registry
+import { AppendOptions, appendTransformer } from './transformers/append';
-
-import { appendTransformer, AppendOptions } from './transformers/append';
 import { reduceTransformer, ReduceTransformerOptions } from './transformers/reduce';
 import { filterFieldsTransformer, filterFramesTransformer } from './transformers/filter';
 import { filterFieldsByNameTransformer, FilterFieldsByNameTransformerOptions } from './transformers/filterByName';
 import { noopTransformer } from './transformers/noop';
-import { DataTransformerInfo, DataTransformerConfig } from '../types/transformations';
+import { DataTransformerConfig, DataTransformerInfo } from '../types/transformations';
 import { filterFramesByRefIdTransformer } from './transformers/filterByRefId';
+import { seriesToColumnsTransformer } from './transformers/seriesToColumns';
+
+// Initalize the Registry
 
 /**
  * Apply configured transformations to the input data
@@ -68,6 +69,7 @@ export const transformersRegistry = new TransformerRegistry(() => [
   filterFramesByRefIdTransformer,
   appendTransformer,
   reduceTransformer,
+  seriesToColumnsTransformer,
 ]);
 
 export { ReduceTransformerOptions, FilterFieldsByNameTransformerOptions };

packages/grafana-data/src/transformations/transformers/ids.ts

@@ -1,9 +1,10 @@
 export enum DataTransformerID {
-  //  join = 'join', // Pick a field and merge all series based on that field
+  // join = 'join', // Pick a field and merge all series based on that field
   append = 'append', // Merge all series together
   //  rotate = 'rotate', // Columns to rows
   reduce = 'reduce', // Run calculations on fields
 
+  seriesToColumns = 'seriesToColumns', // former table transform timeseries_to_columns
   filterFields = 'filterFields', // Pick some fields (keep all frames)
   filterFieldsByName = 'filterFieldsByName', // Pick fields with name matching regex (keep all frames)
   filterFrames = 'filterFrames', // Pick some frames (keep all fields)

packages/grafana-data/src/transformations/transformers/seriesToColumns.test.ts

@@ -0,0 +1,177 @@
+import {
+  ArrayVector,
+  DataTransformerConfig,
+  DataTransformerID,
+  FieldType,
+  toDataFrame,
+  transformDataFrame,
+} from '@grafana/data';
+import { SeriesToColumnsOptions } from './seriesToColumns';
+
+describe('SeriesToColumns Transformer', () => {
+  const everySecondSeries = toDataFrame({
+    name: 'even',
+    fields: [
+      { name: 'time', type: FieldType.time, values: [3000, 4000, 5000, 6000] },
+      { name: 'temperature', type: FieldType.number, values: [10.3, 10.4, 10.5, 10.6] },
+      { name: 'humidity', type: FieldType.number, values: [10000.3, 10000.4, 10000.5, 10000.6] },
+    ],
+  });
+
+  const everyOtherSecondSeries = toDataFrame({
+    name: 'odd',
+    fields: [
+      { name: 'time', type: FieldType.time, values: [1000, 3000, 5000, 7000] },
+      { name: 'temperature', type: FieldType.number, values: [11.1, 11.3, 11.5, 11.7] },
+      { name: 'humidity', type: FieldType.number, values: [11000.1, 11000.3, 11000.5, 11000.7] },
+    ],
+  });
+
+  it('joins by time field', () => {
+    const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
+      id: DataTransformerID.seriesToColumns,
+      options: {
+        byField: 'time',
+      },
+    };
+
+    const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0];
+    expect(filtered.fields).toEqual([
+      {
+        name: 'time',
+        type: FieldType.time,
+        values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
+        config: {},
+        labels: { origin: 'even,odd' },
+      },
+      {
+        name: 'temperature {even}',
+        type: FieldType.number,
+        values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
+        config: {},
+        labels: { origin: 'even' },
+      },
+      {
+        name: 'humidity {even}',
+        type: FieldType.number,
+        values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
+        config: {},
+        labels: { origin: 'even' },
+      },
+      {
+        name: 'temperature {odd}',
+        type: FieldType.number,
+        values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
+        config: {},
+        labels: { origin: 'odd' },
+      },
+      {
+        name: 'humidity {odd}',
+        type: FieldType.number,
+        values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
+        config: {},
+        labels: { origin: 'odd' },
+      },
+    ]);
+  });
+
+  it('joins by temperature field', () => {
+    const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
+      id: DataTransformerID.seriesToColumns,
+      options: {
+        byField: 'temperature',
+      },
+    };
+
+    const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0];
+    expect(filtered.fields).toEqual([
+      {
+        name: 'temperature',
+        type: FieldType.number,
+        values: new ArrayVector([10.3, 10.4, 10.5, 10.6, 11.1, 11.3, 11.5, 11.7]),
+        config: {},
+        labels: { origin: 'even,odd' },
+      },
+      {
+        name: 'time {even}',
+        type: FieldType.time,
+        values: new ArrayVector([3000, 4000, 5000, 6000, null, null, null, null]),
+        config: {},
+        labels: { origin: 'even' },
+      },
+      {
+        name: 'humidity {even}',
+        type: FieldType.number,
+        values: new ArrayVector([10000.3, 10000.4, 10000.5, 10000.6, null, null, null, null]),
+        config: {},
+        labels: { origin: 'even' },
+      },
+      {
+        name: 'time {odd}',
+        type: FieldType.time,
+        values: new ArrayVector([null, null, null, null, 1000, 3000, 5000, 7000]),
+        config: {},
+        labels: { origin: 'odd' },
+      },
+      {
+        name: 'humidity {odd}',
+        type: FieldType.number,
+        values: new ArrayVector([null, null, null, null, 11000.1, 11000.3, 11000.5, 11000.7]),
+        config: {},
+        labels: { origin: 'odd' },
+      },
+    ]);
+  });
+
+  it('joins by time field in reverse order', () => {
+    const cfg: DataTransformerConfig<SeriesToColumnsOptions> = {
+      id: DataTransformerID.seriesToColumns,
+      options: {
+        byField: 'time',
+      },
+    };
+
+    everySecondSeries.fields[0].values = new ArrayVector(everySecondSeries.fields[0].values.toArray().reverse());
+    everySecondSeries.fields[1].values = new ArrayVector(everySecondSeries.fields[1].values.toArray().reverse());
+    everySecondSeries.fields[2].values = new ArrayVector(everySecondSeries.fields[2].values.toArray().reverse());
+
+    const filtered = transformDataFrame([cfg], [everySecondSeries, everyOtherSecondSeries])[0];
+    expect(filtered.fields).toEqual([
+      {
+        name: 'time',
+        type: FieldType.time,
+        values: new ArrayVector([1000, 3000, 4000, 5000, 6000, 7000]),
+        config: {},
+        labels: { origin: 'even,odd' },
+      },
+      {
+        name: 'temperature {even}',
+        type: FieldType.number,
+        values: new ArrayVector([null, 10.3, 10.4, 10.5, 10.6, null]),
+        config: {},
+        labels: { origin: 'even' },
+      },
+      {
+        name: 'humidity {even}',
+        type: FieldType.number,
+        values: new ArrayVector([null, 10000.3, 10000.4, 10000.5, 10000.6, null]),
+        config: {},
+        labels: { origin: 'even' },
+      },
+      {
+        name: 'temperature {odd}',
+        type: FieldType.number,
+        values: new ArrayVector([11.1, 11.3, null, 11.5, null, 11.7]),
+        config: {},
+        labels: { origin: 'odd' },
+      },
+      {
+        name: 'humidity {odd}',
+        type: FieldType.number,
+        values: new ArrayVector([11000.1, 11000.3, null, 11000.5, null, 11000.7]),
+        config: {},
+        labels: { origin: 'odd' },
+      },
+    ]);
+  });
+});

packages/grafana-data/src/transformations/transformers/seriesToColumns.ts

@@ -0,0 +1,123 @@
+import { DataFrame, DataTransformerInfo } from '../../types';
+import { DataTransformerID } from './ids';
+import { MutableDataFrame } from '../../dataframe';
+import { filterFieldsByNameTransformer } from './filterByName';
+import { ArrayVector } from '../../vector';
+
+export interface SeriesToColumnsOptions {
+  byField: string;
+}
+
+export const seriesToColumnsTransformer: DataTransformerInfo<SeriesToColumnsOptions> = {
+  id: DataTransformerID.seriesToColumns,
+  name: 'Series as Columns',
+  description: 'Groups series by field and returns values as columns',
+  defaultOptions: {},
+  transformer: options => (data: DataFrame[]) => {
+    const regex = `/^(${options.byField})$/`;
+    // not sure if I should use filterFieldsByNameTransformer to get the key field
+    const keyDataFrames = filterFieldsByNameTransformer.transformer({ include: regex })(data);
+    if (!keyDataFrames.length) {
+      // for now we only parse data frames with 2 fields
+      return data;
+    }
+
+    // not sure if I should use filterFieldsByNameTransformer to get the other fields
+    const otherDataFrames = filterFieldsByNameTransformer.transformer({ exclude: regex })(data);
+    if (!otherDataFrames.length) {
+      // for now we only parse data frames with 2 fields
+      return data;
+    }
+
+    const processed = new MutableDataFrame();
+    const origins: string[] = [];
+    for (let frameIndex = 0; frameIndex < keyDataFrames.length; frameIndex++) {
+      const frame = keyDataFrames[frameIndex];
+      const origin = getOrigin(frame, frameIndex);
+      origins.push(origin);
+    }
+
+    processed.addField({
+      ...keyDataFrames[0].fields[0],
+      values: new ArrayVector([]),
+      labels: { origin: origins.join(',') },
+    });
+
+    for (let frameIndex = 0; frameIndex < otherDataFrames.length; frameIndex++) {
+      const frame = otherDataFrames[frameIndex];
+      for (let fieldIndex = 0; fieldIndex < frame.fields.length; fieldIndex++) {
+        const field = frame.fields[fieldIndex];
+        const origin = getOrigin(frame, frameIndex);
+        const name = getColumnName(otherDataFrames, frameIndex, fieldIndex, false);
+        if (processed.fields.find(field => field.name === name)) {
+          continue;
+        }
+        processed.addField({ ...field, name, values: new ArrayVector([]), labels: { origin } });
+      }
+    }
+
+    const byKeyField: { [key: string]: { [key: string]: any } } = {};
+    // this loop creates a dictionary object that groups the key fields values
+    /*
+    {
+      "key field first value as string" : {
+        "key field name": key field first value,
+        "other series name": other series value
+        "other series n name": other series n value
+      },
+      "key field n value as string" : {
+        "key field name": key field n value,
+        "other series name": other series value
+        "other series n name": other series n value
+      }
+    }
+     */
+    for (let seriesIndex = 0; seriesIndex < keyDataFrames.length; seriesIndex++) {
+      const keyDataFrame = keyDataFrames[seriesIndex];
+      const keyField = keyDataFrame.fields[0];
+      const keyColumnName = getColumnName(keyDataFrames, seriesIndex, 0, true);
+      const keyValues = keyField.values;
+      for (let valueIndex = 0; valueIndex < keyValues.length; valueIndex++) {
+        const keyValue = keyValues.get(valueIndex);
+        const keyValueAsString = keyValue.toString();
+        if (!byKeyField[keyValueAsString]) {
+          byKeyField[keyValueAsString] = { [keyColumnName]: keyValue };
+        }
+        const otherDataFrame = otherDataFrames[seriesIndex];
+        for (let otherIndex = 0; otherIndex < otherDataFrame.fields.length; otherIndex++) {
+          const otherColumnName = getColumnName(otherDataFrames, seriesIndex, otherIndex, false);
+          const otherField = otherDataFrame.fields[otherIndex];
+          const otherValue = otherField.values.get(valueIndex);
+          if (!byKeyField[keyValueAsString][otherColumnName]) {
+            byKeyField[keyValueAsString] = { ...byKeyField[keyValueAsString], [otherColumnName]: otherValue };
+          }
+        }
+      }
+    }
+
+    const keyValueStrings = Object.keys(byKeyField);
+    for (let rowIndex = 0; rowIndex < keyValueStrings.length; rowIndex++) {
+      const keyValueAsString = keyValueStrings[rowIndex];
+      for (let fieldIndex = 0; fieldIndex < processed.fields.length; fieldIndex++) {
+        const field = processed.fields[fieldIndex];
+        const value = byKeyField[keyValueAsString][field.name] ?? null;
+        field.values.add(value);
+      }
+    }
+
+    return [processed];
+  },
+};
+
+const getColumnName = (frames: DataFrame[], frameIndex: number, fieldIndex: number, isKeyField = false) => {
+  const frame = frames[frameIndex];
+  const frameName = frame.name || `${frameIndex}`;
+  const fieldName = frame.fields[fieldIndex].name;
+  const seriesName = isKeyField ? fieldName : `${fieldName} {${frameName}}`;
+
+  return seriesName;
+};
+
+const getOrigin = (frame: DataFrame, index: number) => {
+  return frame.name || `${index}`;
+};